2024-11-23 22:51:15,400 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-23 22:51:15,418 main DEBUG Took 0.015397 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-23 22:51:15,419 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-23 22:51:15,420 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-23 22:51:15,421 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-23 22:51:15,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,434 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-23 22:51:15,450 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,453 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,455 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,456 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,458 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,459 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,463 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,464 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,465 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,467 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-23 22:51:15,469 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,470 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-23 22:51:15,472 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-23 22:51:15,474 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-23 22:51:15,476 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-23 22:51:15,477 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-23 22:51:15,479 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-23 22:51:15,480 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-23 22:51:15,491 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-23 22:51:15,494 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-23 22:51:15,496 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-23 22:51:15,496 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-23 22:51:15,497 main DEBUG createAppenders(={Console})
2024-11-23 22:51:15,497 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-23 22:51:15,498 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-23 22:51:15,498 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-23 22:51:15,499 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-23 22:51:15,499 main DEBUG OutputStream closed
2024-11-23 22:51:15,500 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-23 22:51:15,500 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-23 22:51:15,500 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-23 22:51:15,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-23 22:51:15,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-23 22:51:15,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-23 22:51:15,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-23 22:51:15,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-23 22:51:15,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-23 22:51:15,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-23 22:51:15,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-23 22:51:15,603 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-23 22:51:15,603 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-23 22:51:15,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-23 22:51:15,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-23 22:51:15,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-23 22:51:15,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-23 22:51:15,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-23 22:51:15,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-23 22:51:15,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-23 22:51:15,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-23 22:51:15,611 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-23 22:51:15,611 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-23 22:51:15,611 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-23 22:51:15,612 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-23T22:51:15,935 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17
2024-11-23 22:51:15,938 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-23 22:51:15,938 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-23T22:51:15,950 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-23T22:51:15,989 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=7687
2024-11-23T22:51:15,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:51:16,007 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5, deleteOnExit=true
2024-11-23T22:51:16,007 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:51:16,008 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/test.cache.data in system properties and HBase conf
2024-11-23T22:51:16,008 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:51:16,009 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:51:16,010 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:51:16,010 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:51:16,010 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:51:16,089 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-23T22:51:16,172 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:51:16,175 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:51:16,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:51:16,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:51:16,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:51:16,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:51:16,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:51:16,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:51:16,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:51:16,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:51:16,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:51:16,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:51:16,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:51:16,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:51:16,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:51:16,647 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:51:17,315 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-23T22:51:17,385 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2709ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-23T22:51:17,452 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:51:17,508 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:51:17,528 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:51:17,528 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:51:17,529 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:51:17,540 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:51:17,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:51:17,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:51:17,725 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de997b9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/java.io.tmpdir/jetty-localhost-46195-hadoop-hdfs-3_4_1-tests_jar-_-any-14567095485486495311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:51:17,731 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:46195}
2024-11-23T22:51:17,732 INFO  [Time-limited test {}] server.Server(415): Started @3056ms
2024-11-23T22:51:17,760 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:51:18,294 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:51:18,304 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:51:18,305 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:51:18,305 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:51:18,305 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:51:18,306 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26c88bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:51:18,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@621a7cbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:51:18,408 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11e88411{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/java.io.tmpdir/jetty-localhost-43635-hadoop-hdfs-3_4_1-tests_jar-_-any-5153803400301157201/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:51:18,409 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75423500{HTTP/1.1, (http/1.1)}{localhost:43635}
2024-11-23T22:51:18,409 INFO  [Time-limited test {}] server.Server(415): Started @3734ms
2024-11-23T22:51:18,458 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:51:18,563 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:51:18,570 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:51:18,574 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:51:18,574 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:51:18,575 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:51:18,576 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43794ae7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:51:18,577 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35f1cf70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:51:18,698 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@590b36b7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/java.io.tmpdir/jetty-localhost-40633-hadoop-hdfs-3_4_1-tests_jar-_-any-16553443149393501066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:51:18,698 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@237fc06a{HTTP/1.1, (http/1.1)}{localhost:40633}
2024-11-23T22:51:18,699 INFO  [Time-limited test {}] server.Server(415): Started @4023ms
2024-11-23T22:51:18,701 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:51:20,633 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data2/current/BP-294770882-172.17.0.2-1732402276733/current, will proceed with Du for space computation calculation, 
2024-11-23T22:51:20,633 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data3/current/BP-294770882-172.17.0.2-1732402276733/current, will proceed with Du for space computation calculation, 
2024-11-23T22:51:20,633 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data4/current/BP-294770882-172.17.0.2-1732402276733/current, will proceed with Du for space computation calculation, 
2024-11-23T22:51:20,633 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data1/current/BP-294770882-172.17.0.2-1732402276733/current, will proceed with Du for space computation calculation, 
2024-11-23T22:51:20,665 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:51:20,665 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:51:20,709 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1d628a5e27081a8 with lease ID 0x91053dd034304b78: Processing first storage report for DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4 from datanode DatanodeRegistration(127.0.0.1:38417, datanodeUuid=c2f8c6ce-6be1-4e37-806a-6817b37d2694, infoPort=46637, infoSecurePort=0, ipcPort=39067, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733)
2024-11-23T22:51:20,710 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1d628a5e27081a8 with lease ID 0x91053dd034304b78: from storage DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4 node DatanodeRegistration(127.0.0.1:38417, datanodeUuid=c2f8c6ce-6be1-4e37-806a-6817b37d2694, infoPort=46637, infoSecurePort=0, ipcPort=39067, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-23T22:51:20,711 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb1ff12a849e7490 with lease ID 0x91053dd034304b79: Processing first storage report for DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9 from datanode DatanodeRegistration(127.0.0.1:40255, datanodeUuid=5a84c4f6-973d-43d9-be1f-fc255b5cbf8e, infoPort=42481, infoSecurePort=0, ipcPort=34421, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733)
2024-11-23T22:51:20,711 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb1ff12a849e7490 with lease ID 0x91053dd034304b79: from storage DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9 node DatanodeRegistration(127.0.0.1:40255, datanodeUuid=5a84c4f6-973d-43d9-be1f-fc255b5cbf8e, infoPort=42481, infoSecurePort=0, ipcPort=34421, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:51:20,711 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1d628a5e27081a8 with lease ID 0x91053dd034304b78: Processing first storage report for DS-a4cac7c1-7c4a-443f-87fc-1518c6c0aa3d from datanode DatanodeRegistration(127.0.0.1:38417, datanodeUuid=c2f8c6ce-6be1-4e37-806a-6817b37d2694, infoPort=46637, infoSecurePort=0, ipcPort=39067, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733)
2024-11-23T22:51:20,711 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1d628a5e27081a8 with lease ID 0x91053dd034304b78: from storage DS-a4cac7c1-7c4a-443f-87fc-1518c6c0aa3d node DatanodeRegistration(127.0.0.1:38417, datanodeUuid=c2f8c6ce-6be1-4e37-806a-6817b37d2694, infoPort=46637, infoSecurePort=0, ipcPort=39067, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:51:20,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb1ff12a849e7490 with lease ID 0x91053dd034304b79: Processing first storage report for DS-9d36f6ce-2b8f-44f7-a5f1-24a0c5cf827e from datanode DatanodeRegistration(127.0.0.1:40255, datanodeUuid=5a84c4f6-973d-43d9-be1f-fc255b5cbf8e, infoPort=42481, infoSecurePort=0, ipcPort=34421, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733)
2024-11-23T22:51:20,712 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb1ff12a849e7490 with lease ID 0x91053dd034304b79: from storage DS-9d36f6ce-2b8f-44f7-a5f1-24a0c5cf827e node DatanodeRegistration(127.0.0.1:40255, datanodeUuid=5a84c4f6-973d-43d9-be1f-fc255b5cbf8e, infoPort=42481, infoSecurePort=0, ipcPort=34421, storageInfo=lv=-57;cid=testClusterID;nsid=787760453;c=1732402276733), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:51:20,730 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17
2024-11-23T22:51:20,793 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/zookeeper_0, clientPort=49185, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:51:20,803 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49185
2024-11-23T22:51:20,813 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:20,815 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:21,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:51:21,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:51:21,432 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8 with version=8
2024-11-23T22:51:21,432 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:51:21,508 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-23T22:51:21,731 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:51:21,740 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:21,741 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:21,745 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:51:21,745 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:21,745 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:51:21,866 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:51:21,916 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-23T22:51:21,925 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-23T22:51:21,928 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:51:21,949 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 61298 (auto-detected)
2024-11-23T22:51:21,950 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-23T22:51:21,970 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33503
2024-11-23T22:51:21,995 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33503 connecting to ZooKeeper ensemble=127.0.0.1:49185
2024-11-23T22:51:22,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335030x0, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:51:22,127 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33503-0x10169e4f4f30000 connected
2024-11-23T22:51:22,217 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:22,221 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:22,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:51:22,236 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8, hbase.cluster.distributed=false
2024-11-23T22:51:22,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:51:22,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33503
2024-11-23T22:51:22,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33503
2024-11-23T22:51:22,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33503
2024-11-23T22:51:22,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33503
2024-11-23T22:51:22,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33503
2024-11-23T22:51:22,357 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:51:22,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:22,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:22,359 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:51:22,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:51:22,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:51:22,362 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:51:22,364 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:51:22,365 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38141
2024-11-23T22:51:22,367 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38141 connecting to ZooKeeper ensemble=127.0.0.1:49185
2024-11-23T22:51:22,368 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:22,372 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:22,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381410x0, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:51:22,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381410x0, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:51:22,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38141-0x10169e4f4f30001 connected
2024-11-23T22:51:22,394 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:51:22,401 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:51:22,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:51:22,408 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:51:22,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38141
2024-11-23T22:51:22,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38141
2024-11-23T22:51:22,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38141
2024-11-23T22:51:22,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38141
2024-11-23T22:51:22,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38141
2024-11-23T22:51:22,427 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:33503
2024-11-23T22:51:22,428 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:22,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:51:22,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:51:22,447 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:22,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:51:22,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:22,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:22,477 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:51:22,479 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,33503,1732402281583 from backup master directory
2024-11-23T22:51:22,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:22,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:51:22,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:51:22,487 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:51:22,488 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:22,490 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-23T22:51:22,491 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-23T22:51:22,545 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase.id] with ID: bca42c44-d766-4d23-9dad-4e496c3cf280
2024-11-23T22:51:22,546 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/.tmp/hbase.id
2024-11-23T22:51:22,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:51:22,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:51:22,563 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/.tmp/hbase.id]:[hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase.id]
2024-11-23T22:51:22,605 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:22,609 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:51:22,627 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-23T22:51:22,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:22,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:22,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:51:22,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:51:22,663 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:51:22,665 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:51:22,670 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:51:22,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:51:22,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:51:22,717 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store
2024-11-23T22:51:22,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:51:22,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:51:22,741 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-23T22:51:22,744 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:22,746 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:51:22,746 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:51:22,746 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:51:22,747 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:51:22,748 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:51:22,748 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:51:22,749 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402282746Disabling compacts and flushes for region at 1732402282746Disabling writes for close at 1732402282748 (+2 ms)Writing region close event to WAL at 1732402282748Closed at 1732402282748
2024-11-23T22:51:22,752 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/.initializing
2024-11-23T22:51:22,752 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/WALs/dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:22,775 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C33503%2C1732402281583, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/WALs/dd96e8dc8369,33503,1732402281583, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/oldWALs, maxLogs=10
2024-11-23T22:51:22,785 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C33503%2C1732402281583.1732402282781
2024-11-23T22:51:22,805 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/WALs/dd96e8dc8369,33503,1732402281583/dd96e8dc8369%2C33503%2C1732402281583.1732402282781
2024-11-23T22:51:22,815 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:51:22,816 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:51:22,816 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:22,819 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,820 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,853 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:51:22,874 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:51:22,877 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:22,879 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:22,880 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:51:22,883 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:51:22,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:22,885 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:51:22,885 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:51:22,888 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:51:22,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:22,889 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:51:22,889 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:51:22,892 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:51:22,892 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:22,893 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:51:22,893 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,897 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,898 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,904 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,904 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,908 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:51:22,912 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:51:22,917 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:51:22,918 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850934, jitterRate=0.08201910555362701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:51:22,925 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402282832Initializing all the Stores at 1732402282834 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402282834Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402282835 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402282835Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402282835Cleaning up temporary data from old regions at 1732402282904 (+69 ms)Region opened successfully at 1732402282924 (+20 ms)
2024-11-23T22:51:22,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:51:22,956 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2786bbd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:51:22,982 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:51:22,991 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:51:22,992 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:51:22,994 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:51:22,995 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-23T22:51:23,000 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-23T22:51:23,000 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:51:23,023 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:51:23,031 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:51:23,042 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:51:23,044 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:51:23,046 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:51:23,055 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:51:23,058 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:51:23,062 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:51:23,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:51:23,145 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:51:23,158 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:51:23,181 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:51:23,192 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:51:23,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:51:23,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:51:23,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,206 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,33503,1732402281583, sessionid=0x10169e4f4f30000, setting cluster-up flag (Was=false)
2024-11-23T22:51:23,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,276 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:51:23,279 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:23,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:23,329 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:51:23,334 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:23,342 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:51:23,405 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:51:23,414 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:51:23,418 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(746): ClusterId : bca42c44-d766-4d23-9dad-4e496c3cf280
2024-11-23T22:51:23,420 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:51:23,420 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:51:23,425 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,33503,1732402281583
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:51:23,436 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:51:23,436 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:51:23,438 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,439 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402313439
2024-11-23T22:51:23,440 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:51:23,441 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:51:23,443 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:51:23,444 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:51:23,445 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:51:23,445 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:51:23,446 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:51:23,446 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:51:23,446 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:51:23,447 DEBUG [RS:0;dd96e8dc8369:38141 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbbc13f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:51:23,447 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,451 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:23,451 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:51:23,451 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:51:23,452 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:51:23,453 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:51:23,455 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:51:23,456 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:51:23,457 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402283457,5,FailOnTimeoutGroup]
2024-11-23T22:51:23,458 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402283457,5,FailOnTimeoutGroup]
2024-11-23T22:51:23,458 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,458 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:51:23,459 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,460 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,461 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:38141
2024-11-23T22:51:23,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:51:23,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:51:23,465 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:51:23,465 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:51:23,465 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:51:23,466 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:51:23,466 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8
2024-11-23T22:51:23,467 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,33503,1732402281583 with port=38141, startcode=1732402282326
2024-11-23T22:51:23,479 DEBUG [RS:0;dd96e8dc8369:38141 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:51:23,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:51:23,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:51:23,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:23,485 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:51:23,488 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:51:23,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:23,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:23,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:51:23,492 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:51:23,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:23,493 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:23,494 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:51:23,496 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:51:23,496 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:23,497 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:23,498 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:51:23,500 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:51:23,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:23,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:23,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:51:23,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740
2024-11-23T22:51:23,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740
2024-11-23T22:51:23,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:51:23,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:51:23,507 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:51:23,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:51:23,513 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:51:23,515 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733995, jitterRate=-0.06667780876159668}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:51:23,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402283483Initializing all the Stores at 1732402283485 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402283485Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402283485Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402283485Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402283485Cleaning up temporary data from old regions at 1732402283506 (+21 ms)Region opened successfully at 1732402283520 (+14 ms)
2024-11-23T22:51:23,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:51:23,520 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:51:23,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:51:23,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:51:23,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:51:23,522 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:51:23,522 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402283520Disabling compacts and flushes for region at 1732402283520Disabling writes for close at 1732402283521 (+1 ms)Writing region close event to WAL at 1732402283522 (+1 ms)Closed at 1732402283522
2024-11-23T22:51:23,526 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:51:23,526 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:51:23,532 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:51:23,540 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:51:23,542 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:51:23,551 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51999, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:51:23,556 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33503 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,559 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33503 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,572 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8
2024-11-23T22:51:23,572 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35437
2024-11-23T22:51:23,572 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:51:23,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:51:23,582 DEBUG [RS:0;dd96e8dc8369:38141 {}] zookeeper.ZKUtil(111): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,582 WARN  [RS:0;dd96e8dc8369:38141 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:51:23,582 INFO  [RS:0;dd96e8dc8369:38141 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:51:23,583 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,585 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,38141,1732402282326]
2024-11-23T22:51:23,611 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:51:23,627 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:51:23,631 INFO  [RS:0;dd96e8dc8369:38141 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:51:23,632 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,633 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:51:23,638 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:51:23,640 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,640 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,640 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,640 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,641 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,642 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,642 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:51:23,642 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:51:23,642 DEBUG [RS:0;dd96e8dc8369:38141 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:51:23,643 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,643 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,643 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,644 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,644 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,644 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,38141,1732402282326-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:51:23,663 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:51:23,666 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,38141,1732402282326-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,666 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,666 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.Replication(171): dd96e8dc8369,38141,1732402282326 started
2024-11-23T22:51:23,685 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:23,685 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,38141,1732402282326, RpcServer on dd96e8dc8369/172.17.0.2:38141, sessionid=0x10169e4f4f30001
2024-11-23T22:51:23,686 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:51:23,686 DEBUG [RS:0;dd96e8dc8369:38141 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,687 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,38141,1732402282326'
2024-11-23T22:51:23,687 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:51:23,688 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:51:23,689 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:51:23,689 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:51:23,689 DEBUG [RS:0;dd96e8dc8369:38141 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,689 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,38141,1732402282326'
2024-11-23T22:51:23,689 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:51:23,690 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:51:23,691 DEBUG [RS:0;dd96e8dc8369:38141 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:51:23,691 INFO  [RS:0;dd96e8dc8369:38141 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:51:23,691 INFO  [RS:0;dd96e8dc8369:38141 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:51:23,693 WARN  [dd96e8dc8369:33503 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:51:23,802 INFO  [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C38141%2C1732402282326, suffix=, logDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs, maxLogs=32
2024-11-23T22:51:23,806 INFO  [RS:0;dd96e8dc8369:38141 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402283805
2024-11-23T22:51:23,814 INFO  [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402283805
2024-11-23T22:51:23,815 DEBUG [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42481:42481),(127.0.0.1/127.0.0.1:46637:46637)]
2024-11-23T22:51:23,946 DEBUG [dd96e8dc8369:33503 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:51:23,956 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:23,961 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,38141,1732402282326, state=OPENING
2024-11-23T22:51:24,013 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:51:24,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:24,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:51:24,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:51:24,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:51:24,026 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:51:24,030 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,38141,1732402282326}]
2024-11-23T22:51:24,203 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:51:24,206 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33441, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:51:24,216 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:51:24,217 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:51:24,220 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C38141%2C1732402282326.meta, suffix=.meta, logDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326, archiveDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs, maxLogs=32
2024-11-23T22:51:24,222 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.meta.1732402284222.meta
2024-11-23T22:51:24,230 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.meta.1732402284222.meta
2024-11-23T22:51:24,240 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:51:24,241 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:51:24,243 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:51:24,245 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:51:24,250 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:51:24,255 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:51:24,255 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:24,256 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:51:24,256 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:51:24,259 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:51:24,261 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:51:24,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:24,262 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:24,262 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:51:24,264 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:51:24,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:24,265 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:24,265 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:51:24,267 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:51:24,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:24,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:24,269 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:51:24,270 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:51:24,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:24,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:51:24,272 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:51:24,273 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740
2024-11-23T22:51:24,276 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740
2024-11-23T22:51:24,279 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:51:24,279 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:51:24,280 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:51:24,283 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:51:24,285 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754820, jitterRate=-0.040197908878326416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:51:24,285 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:51:24,287 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402284257Writing region info on filesystem at 1732402284257Initializing all the Stores at 1732402284259 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402284259Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402284259Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402284259Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402284259Cleaning up temporary data from old regions at 1732402284279 (+20 ms)Running coprocessor post-open hooks at 1732402284285 (+6 ms)Region opened successfully at 1732402284286 (+1 ms)
2024-11-23T22:51:24,293 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402284195
2024-11-23T22:51:24,303 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:51:24,303 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:51:24,305 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:24,307 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,38141,1732402282326, state=OPEN
2024-11-23T22:51:24,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:51:24,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:51:24,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:51:24,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:51:24,392 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:24,401 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:51:24,402 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,38141,1732402282326 in 363 msec
2024-11-23T22:51:24,409 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:51:24,409 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 872 msec
2024-11-23T22:51:24,410 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:51:24,410 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:51:24,427 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:51:24,428 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,38141,1732402282326, seqNum=-1]
2024-11-23T22:51:24,449 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:51:24,451 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:51:24,474 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1050 sec
2024-11-23T22:51:24,474 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402284474, completionTime=-1
2024-11-23T22:51:24,476 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:51:24,477 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:51:24,499 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:51:24,499 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402344499
2024-11-23T22:51:24,500 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402404499
2024-11-23T22:51:24,500 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec
2024-11-23T22:51:24,502 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,502 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,502 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,504 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:33503, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,504 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,504 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,510 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:51:24,530 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.042sec
2024-11-23T22:51:24,531 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:51:24,532 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:51:24,533 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:51:24,533 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:51:24,534 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:51:24,535 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:51:24,535 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:51:24,544 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:51:24,545 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:51:24,546 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33503,1732402281583-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:51:24,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306ff804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:51:24,647 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-23T22:51:24,648 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-23T22:51:24,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,33503,-1 for getting cluster id
2024-11-23T22:51:24,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:51:24,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bca42c44-d766-4d23-9dad-4e496c3cf280'
2024-11-23T22:51:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:51:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bca42c44-d766-4d23-9dad-4e496c3cf280"

2024-11-23T22:51:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aeed19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:51:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,33503,-1]
2024-11-23T22:51:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:51:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:51:24,677 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:51:24,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129e34bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:51:24,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:51:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,38141,1732402282326, seqNum=-1]
2024-11-23T22:51:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:51:24,693 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:51:24,711 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:24,712 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:51:24,718 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:51:24,722 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-23T22:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,33503,1732402281583
2024-11-23T22:51:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cdd6981
2024-11-23T22:51:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-23T22:51:24,732 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-23T22:51:24,734 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-23T22:51:24,734 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-23T22:51:24,737 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:51:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-23T22:51:24,748 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-23T22:51:24,750 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-23T22:51:24,751 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:24,753 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-23T22:51:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:51:24,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741835_1011 (size=389)
2024-11-23T22:51:24,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741835_1011 (size=389)
2024-11-23T22:51:25,196 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffe6ec725dba0aad1ddfbd143a15baa0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8
2024-11-23T22:51:25,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741836_1012 (size=72)
2024-11-23T22:51:25,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741836_1012 (size=72)
2024-11-23T22:51:25,206 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:25,207 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing ffe6ec725dba0aad1ddfbd143a15baa0, disabling compactions & flushes
2024-11-23T22:51:25,207 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,207 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,207 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0. after waiting 0 ms
2024-11-23T22:51:25,207 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,207 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,207 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffe6ec725dba0aad1ddfbd143a15baa0:
Waiting for close lock at 1732402285207Disabling compacts and flushes for region at 1732402285207Disabling writes for close at 1732402285207Writing region close event to WAL at 1732402285207Closed at 1732402285207
2024-11-23T22:51:25,209 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-23T22:51:25,213 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732402285209"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402285209"}]},"ts":"1732402285209"}
2024-11-23T22:51:25,218 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-23T22:51:25,221 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-23T22:51:25,223 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402285221"}]},"ts":"1732402285221"}
2024-11-23T22:51:25,228 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-23T22:51:25,229 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=ffe6ec725dba0aad1ddfbd143a15baa0, ASSIGN}]
2024-11-23T22:51:25,232 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=ffe6ec725dba0aad1ddfbd143a15baa0, ASSIGN
2024-11-23T22:51:25,234 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=ffe6ec725dba0aad1ddfbd143a15baa0, ASSIGN; state=OFFLINE, location=dd96e8dc8369,38141,1732402282326; forceNewPlan=false, retain=false
2024-11-23T22:51:25,385 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ffe6ec725dba0aad1ddfbd143a15baa0, regionState=OPENING, regionLocation=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:25,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=ffe6ec725dba0aad1ddfbd143a15baa0, ASSIGN because future has completed
2024-11-23T22:51:25,391 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe6ec725dba0aad1ddfbd143a15baa0, server=dd96e8dc8369,38141,1732402282326}]
2024-11-23T22:51:25,553 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,553 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ffe6ec725dba0aad1ddfbd143a15baa0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:51:25,553 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,554 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:51:25,554 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,554 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,556 INFO  [StoreOpener-ffe6ec725dba0aad1ddfbd143a15baa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ffe6ec725dba0aad1ddfbd143a15baa0 
2024-11-23T22:51:25,559 INFO  [StoreOpener-ffe6ec725dba0aad1ddfbd143a15baa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffe6ec725dba0aad1ddfbd143a15baa0 columnFamilyName info
2024-11-23T22:51:25,559 DEBUG [StoreOpener-ffe6ec725dba0aad1ddfbd143a15baa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:51:25,560 INFO  [StoreOpener-ffe6ec725dba0aad1ddfbd143a15baa0-1 {}] regionserver.HStore(327): Store=ffe6ec725dba0aad1ddfbd143a15baa0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:51:25,561 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,562 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,563 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,563 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,563 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,566 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,569 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:51:25,570 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ffe6ec725dba0aad1ddfbd143a15baa0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811057, jitterRate=0.0313129723072052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:51:25,571 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:25,572 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ffe6ec725dba0aad1ddfbd143a15baa0:
Running coprocessor pre-open hook at 1732402285554Writing region info on filesystem at 1732402285554Initializing all the Stores at 1732402285556 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402285556Cleaning up temporary data from old regions at 1732402285563 (+7 ms)Running coprocessor post-open hooks at 1732402285571 (+8 ms)Region opened successfully at 1732402285571
2024-11-23T22:51:25,574 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0., pid=6, masterSystemTime=1732402285546
2024-11-23T22:51:25,578 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,578 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:25,580 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ffe6ec725dba0aad1ddfbd143a15baa0, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,38141,1732402282326
2024-11-23T22:51:25,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffe6ec725dba0aad1ddfbd143a15baa0, server=dd96e8dc8369,38141,1732402282326 because future has completed
2024-11-23T22:51:25,593 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-23T22:51:25,593 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ffe6ec725dba0aad1ddfbd143a15baa0, server=dd96e8dc8369,38141,1732402282326 in 196 msec
2024-11-23T22:51:25,597 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-23T22:51:25,598 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=ffe6ec725dba0aad1ddfbd143a15baa0, ASSIGN in 364 msec
2024-11-23T22:51:25,599 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-23T22:51:25,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402285600"}]},"ts":"1732402285600"}
2024-11-23T22:51:25,604 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-23T22:51:25,606 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-23T22:51:25,611 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 866 msec
2024-11-23T22:51:29,766 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-23T22:51:29,811 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:51:29,812 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-23T22:51:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:51:31,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-23T22:51:31,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-23T22:51:31,918 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-23T22:51:31,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:51:31,920 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-23T22:51:31,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-23T22:51:31,920 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-23T22:51:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:51:34,857 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-23T22:51:34,860 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-23T22:51:34,866 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-23T22:51:34,867 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:51:34,868 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402294868
2024-11-23T22:51:34,876 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:34,876 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:34,876 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:34,877 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:34,877 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:34,877 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402283805 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402294868
2024-11-23T22:51:34,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:51:34,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402283805 is not closed yet, will try archiving it next time
2024-11-23T22:51:34,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741833_1009 (size=451)
2024-11-23T22:51:34,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741833_1009 (size=451)
2024-11-23T22:51:34,882 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402283805 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402283805
2024-11-23T22:51:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0., hostname=dd96e8dc8369,38141,1732402282326, seqNum=2]
2024-11-23T22:51:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38141 {}] regionserver.HRegion(8855): Flush requested on ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:51:46,927 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe6ec725dba0aad1ddfbd143a15baa0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:51:46,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/9cfadd1382b741db8c446e7611ff2ba4 is 1080, key is row0001/info:/1732402294892/Put/seqid=0
2024-11-23T22:51:46,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741838_1014 (size=12509)
2024-11-23T22:51:46,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741838_1014 (size=12509)
2024-11-23T22:51:46,997 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/9cfadd1382b741db8c446e7611ff2ba4
2024-11-23T22:51:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/9cfadd1382b741db8c446e7611ff2ba4 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4
2024-11-23T22:51:47,052 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4, entries=7, sequenceid=11, filesize=12.2 K
2024-11-23T22:51:47,060 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 133ms, sequenceid=11, compaction requested=false
2024-11-23T22:51:47,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe6ec725dba0aad1ddfbd143a15baa0:

2024-11-23T22:51:50,728 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:51:54,948 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402314947
2024-11-23T22:51:55,159 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:51:55,159 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:55,159 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:55,159 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:55,160 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:55,160 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:51:55,160 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402294868 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402314947
2024-11-23T22:51:55,161 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42481:42481),(127.0.0.1/127.0.0.1:46637:46637)]
2024-11-23T22:51:55,161 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402294868 is not closed yet, will try archiving it next time
2024-11-23T22:51:55,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741837_1013 (size=12399)
2024-11-23T22:51:55,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741837_1013 (size=12399)
2024-11-23T22:51:55,365 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:51:57,572 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:51:59,778 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:01,983 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38141 {}] regionserver.HRegion(8855): Flush requested on ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:52:01,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe6ec725dba0aad1ddfbd143a15baa0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:52:02,186 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:02,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/2fb69f30c4e44938988124663ef3e39e is 1080, key is row0008/info:/1732402308928/Put/seqid=0
2024-11-23T22:52:02,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741840_1016 (size=12509)
2024-11-23T22:52:02,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741840_1016 (size=12509)
2024-11-23T22:52:02,204 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/2fb69f30c4e44938988124663ef3e39e
2024-11-23T22:52:02,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/2fb69f30c4e44938988124663ef3e39e as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e
2024-11-23T22:52:02,226 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e, entries=7, sequenceid=21, filesize=12.2 K
2024-11-23T22:52:02,585 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 358 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:02,586 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 602ms, sequenceid=21, compaction requested=false
2024-11-23T22:52:02,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe6ec725dba0aad1ddfbd143a15baa0:

2024-11-23T22:52:02,587 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-23T22:52:02,587 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4 because midkey is the same as first or last row
2024-11-23T22:52:04,188 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:05,311 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-23T22:52:05,311 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-23T22:52:06,394 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:06,398 WARN  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:06,400 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C38141%2C1732402282326:(num 1732402314947) roll requested
2024-11-23T22:52:06,401 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402326400
2024-11-23T22:52:06,608 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:06,608 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:06,608 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:06,608 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:06,608 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:06,608 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:06,609 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402314947 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402326400
2024-11-23T22:52:06,610 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:52:06,610 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402314947 is not closed yet, will try archiving it next time
2024-11-23T22:52:06,610 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402294868 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402294868
2024-11-23T22:52:06,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741839_1015 (size=7739)
2024-11-23T22:52:06,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741839_1015 (size=7739)
2024-11-23T22:52:08,599 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:10,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region ffe6ec725dba0aad1ddfbd143a15baa0, had cached 0 bytes from a total of 25018
2024-11-23T22:52:10,803 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:13,012 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:15,218 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:17,221 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-23T22:52:17,221 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402337221
2024-11-23T22:52:20,728 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:52:22,229 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:22,231 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:22,231 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C38141%2C1732402282326:(num 1732402337221) roll requested
2024-11-23T22:52:22,231 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:22,231 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:22,232 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:22,232 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:22,232 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:22,232 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402326400 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402337221
2024-11-23T22:52:22,233 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:52:22,233 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402326400 is not closed yet, will try archiving it next time
2024-11-23T22:52:22,234 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402342233
2024-11-23T22:52:22,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741841_1017 (size=4753)
2024-11-23T22:52:22,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741841_1017 (size=4753)
2024-11-23T22:52:27,278 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5042 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:27,278 WARN  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5042 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38141 {}] regionserver.HRegion(8855): Flush requested on ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:52:27,279 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe6ec725dba0aad1ddfbd143a15baa0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:52:27,285 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5048 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:27,286 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5048 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:29,280 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-23T22:52:32,330 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5049 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:32,330 WARN  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5049 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK], DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK]]
2024-11-23T22:52:32,331 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:32,331 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:32,332 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:32,332 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:32,333 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:32,333 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402337221 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402342233
2024-11-23T22:52:32,336 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42481:42481),(127.0.0.1/127.0.0.1:46637:46637)]
2024-11-23T22:52:32,336 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402337221 is not closed yet, will try archiving it next time
2024-11-23T22:52:32,337 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C38141%2C1732402282326:(num 1732402342233) roll requested
2024-11-23T22:52:32,338 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402352337
2024-11-23T22:52:32,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741842_1018 (size=1569)
2024-11-23T22:52:32,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741842_1018 (size=1569)
2024-11-23T22:52:32,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/bb39cf3a1a244435a4859a0479b92240 is 1080, key is row0015/info:/1732402323986/Put/seqid=0
2024-11-23T22:52:32,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741844_1020 (size=12509)
2024-11-23T22:52:32,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741844_1020 (size=12509)
2024-11-23T22:52:32,351 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/bb39cf3a1a244435a4859a0479b92240
2024-11-23T22:52:32,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/bb39cf3a1a244435a4859a0479b92240 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240
2024-11-23T22:52:32,371 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240, entries=7, sequenceid=31, filesize=12.2 K
2024-11-23T22:52:37,373 INFO  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:37,374 WARN  [FSHLog-0-hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8-prefix:dd96e8dc8369,38141,1732402282326 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:37,374 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 10095ms, sequenceid=31, compaction requested=true
2024-11-23T22:52:37,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe6ec725dba0aad1ddfbd143a15baa0:

2024-11-23T22:52:37,374 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-23T22:52:37,374 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4 because midkey is the same as first or last row
2024-11-23T22:52:37,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffe6ec725dba0aad1ddfbd143a15baa0:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:52:37,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:52:37,380 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:52:37,383 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:52:37,385 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HStore(1541): ffe6ec725dba0aad1ddfbd143a15baa0/info is initiating minor compaction (all files)
2024-11-23T22:52:37,385 INFO  [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffe6ec725dba0aad1ddfbd143a15baa0/info in TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:37,386 INFO  [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240] into tmpdir=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp, totalSize=36.6 K
2024-11-23T22:52:37,387 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfadd1382b741db8c446e7611ff2ba4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732402294892
2024-11-23T22:52:37,388 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb69f30c4e44938988124663ef3e39e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732402308928
2024-11-23T22:52:37,388 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb39cf3a1a244435a4859a0479b92240, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732402323986
2024-11-23T22:52:37,405 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5064 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:37,405 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5064 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40255,DS-fd79f301-98ac-4eaf-8245-bcb06ade8ad9,DISK], DatanodeInfoWithStorage[127.0.0.1:38417,DS-26cf9f34-88c3-4645-b8f5-1fbe9f0b14e4,DISK]]
2024-11-23T22:52:37,406 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,406 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,406 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,406 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,406 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,406 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402342233 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402352337
2024-11-23T22:52:37,408 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42481:42481),(127.0.0.1/127.0.0.1:46637:46637)]
2024-11-23T22:52:37,408 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402342233 is not closed yet, will try archiving it next time
2024-11-23T22:52:37,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741843_1019 (size=438)
2024-11-23T22:52:37,408 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402314947 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402314947
2024-11-23T22:52:37,408 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C38141%2C1732402282326:(num 1732402357408) roll requested
2024-11-23T22:52:37,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741843_1019 (size=438)
2024-11-23T22:52:37,409 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402357408
2024-11-23T22:52:37,410 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402326400 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402326400
2024-11-23T22:52:37,411 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402337221 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402337221
2024-11-23T22:52:37,413 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402342233 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402342233
2024-11-23T22:52:37,417 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,417 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,417 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,418 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,418 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,418 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402352337 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402357408
2024-11-23T22:52:37,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741845_1021 (size=93)
2024-11-23T22:52:37,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741845_1021 (size=93)
2024-11-23T22:52:37,421 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:52:37,421 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402352337 is not closed yet, will try archiving it next time
2024-11-23T22:52:37,421 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C38141%2C1732402282326.1732402357421
2024-11-23T22:52:37,422 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402352337 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs/dd96e8dc8369%2C38141%2C1732402282326.1732402352337
2024-11-23T22:52:37,422 INFO  [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffe6ec725dba0aad1ddfbd143a15baa0#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:52:37,424 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/ed4ccc0fb6e547a5aeef437974ce1024 is 1080, key is row0001/info:/1732402294892/Put/seqid=0
2024-11-23T22:52:37,432 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,432 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,432 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,432 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,432 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:37,433 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402357408 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/WALs/dd96e8dc8369,38141,1732402282326/dd96e8dc8369%2C38141%2C1732402282326.1732402357421
2024-11-23T22:52:37,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741848_1024 (size=27710)
2024-11-23T22:52:37,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741848_1024 (size=27710)
2024-11-23T22:52:37,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741846_1022 (size=1258)
2024-11-23T22:52:37,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741846_1022 (size=1258)
2024-11-23T22:52:37,437 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46637:46637),(127.0.0.1/127.0.0.1:42481:42481)]
2024-11-23T22:52:37,443 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/ed4ccc0fb6e547a5aeef437974ce1024 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/ed4ccc0fb6e547a5aeef437974ce1024
2024-11-23T22:52:37,458 INFO  [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffe6ec725dba0aad1ddfbd143a15baa0/info of ffe6ec725dba0aad1ddfbd143a15baa0 into ed4ccc0fb6e547a5aeef437974ce1024(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:52:37,459 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffe6ec725dba0aad1ddfbd143a15baa0:

2024-11-23T22:52:37,460 INFO  [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0., storeName=ffe6ec725dba0aad1ddfbd143a15baa0/info, priority=13, startTime=1732402357377; duration=0sec
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/ed4ccc0fb6e547a5aeef437974ce1024 because midkey is the same as first or last row
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/ed4ccc0fb6e547a5aeef437974ce1024 because midkey is the same as first or last row
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/ed4ccc0fb6e547a5aeef437974ce1024 because midkey is the same as first or last row
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:52:37,461 DEBUG [RS:0;dd96e8dc8369:38141-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffe6ec725dba0aad1ddfbd143a15baa0:info
2024-11-23T22:52:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38141 {}] regionserver.HRegion(8855): Flush requested on ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:52:49,457 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffe6ec725dba0aad1ddfbd143a15baa0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:52:49,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/5cb7dae394574b56a81f0456e6258a23 is 1080, key is row0022/info:/1732402357422/Put/seqid=0
2024-11-23T22:52:49,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741849_1025 (size=12509)
2024-11-23T22:52:49,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741849_1025 (size=12509)
2024-11-23T22:52:49,476 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/5cb7dae394574b56a81f0456e6258a23
2024-11-23T22:52:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/5cb7dae394574b56a81f0456e6258a23 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/5cb7dae394574b56a81f0456e6258a23
2024-11-23T22:52:49,503 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/5cb7dae394574b56a81f0456e6258a23, entries=7, sequenceid=42, filesize=12.2 K
2024-11-23T22:52:49,504 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 47ms, sequenceid=42, compaction requested=false
2024-11-23T22:52:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffe6ec725dba0aad1ddfbd143a15baa0:

2024-11-23T22:52:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-23T22:52:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:52:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/ed4ccc0fb6e547a5aeef437974ce1024 because midkey is the same as first or last row
2024-11-23T22:52:50,729 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:52:55,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region ffe6ec725dba0aad1ddfbd143a15baa0, had cached 0 bytes from a total of 40219
2024-11-23T22:52:57,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:52:57,477 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:52:57,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:52:57,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:52:57,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:52:57,488 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:52:57,489 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:52:57,489 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=245640440, stopped=false
2024-11-23T22:52:57,489 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,33503,1732402281583
2024-11-23T22:52:57,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:52:57,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:52:57,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:52:57,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:52:57,564 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:52:57,565 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:52:57,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:52:57,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:52:57,566 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:52:57,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:52:57,567 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,38141,1732402282326' *****
2024-11-23T22:52:57,568 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:52:57,568 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:52:57,569 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:52:57,569 INFO  [RS:0;dd96e8dc8369:38141 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:52:57,569 INFO  [RS:0;dd96e8dc8369:38141 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:52:57,569 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(3091): Received CLOSE for ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:52:57,570 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,38141,1732402282326
2024-11-23T22:52:57,570 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:52:57,571 INFO  [RS:0;dd96e8dc8369:38141 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:38141.
2024-11-23T22:52:57,571 DEBUG [RS:0;dd96e8dc8369:38141 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:52:57,571 DEBUG [RS:0;dd96e8dc8369:38141 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:52:57,571 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffe6ec725dba0aad1ddfbd143a15baa0, disabling compactions & flushes
2024-11-23T22:52:57,571 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:52:57,571 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:57,571 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:52:57,571 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:57,572 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:52:57,572 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0. after waiting 0 ms
2024-11-23T22:52:57,572 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:57,572 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:52:57,572 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ffe6ec725dba0aad1ddfbd143a15baa0 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-23T22:52:57,572 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-23T22:52:57,573 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:52:57,573 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ffe6ec725dba0aad1ddfbd143a15baa0=TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.}
2024-11-23T22:52:57,573 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:52:57,573 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:52:57,573 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:52:57,573 DEBUG [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ffe6ec725dba0aad1ddfbd143a15baa0
2024-11-23T22:52:57,573 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:52:57,574 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-23T22:52:57,580 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/8fd7d5e0b7b04dd6af149538a1598bc0 is 1080, key is row0029/info:/1732402371459/Put/seqid=0
2024-11-23T22:52:57,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741850_1026 (size=8193)
2024-11-23T22:52:57,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741850_1026 (size=8193)
2024-11-23T22:52:57,588 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/8fd7d5e0b7b04dd6af149538a1598bc0
2024-11-23T22:52:57,597 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/info/6e8131678a2f4a938c558072e89ca80f is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0./info:regioninfo/1732402285579/Put/seqid=0
2024-11-23T22:52:57,599 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/.tmp/info/8fd7d5e0b7b04dd6af149538a1598bc0 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/8fd7d5e0b7b04dd6af149538a1598bc0
2024-11-23T22:52:57,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741851_1027 (size=7016)
2024-11-23T22:52:57,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741851_1027 (size=7016)
2024-11-23T22:52:57,604 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/info/6e8131678a2f4a938c558072e89ca80f
2024-11-23T22:52:57,608 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/8fd7d5e0b7b04dd6af149538a1598bc0, entries=3, sequenceid=48, filesize=8.0 K
2024-11-23T22:52:57,610 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 37ms, sequenceid=48, compaction requested=true
2024-11-23T22:52:57,610 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240] to archive
2024-11-23T22:52:57,613 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:52:57,616 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/9cfadd1382b741db8c446e7611ff2ba4
2024-11-23T22:52:57,618 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/2fb69f30c4e44938988124663ef3e39e
2024-11-23T22:52:57,620 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240 to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/info/bb39cf3a1a244435a4859a0479b92240
2024-11-23T22:52:57,629 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/ns/72925dddfdbe4df08a21f040ce4ccc66 is 43, key is default/ns:d/1732402284454/Put/seqid=0
2024-11-23T22:52:57,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741852_1028 (size=5153)
2024-11-23T22:52:57,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741852_1028 (size=5153)
2024-11-23T22:52:57,636 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/ns/72925dddfdbe4df08a21f040ce4ccc66
2024-11-23T22:52:57,632 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=dd96e8dc8369:33503 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-23T22:52:57,637 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [9cfadd1382b741db8c446e7611ff2ba4=12509, 2fb69f30c4e44938988124663ef3e39e=12509, bb39cf3a1a244435a4859a0479b92240=12509]
2024-11-23T22:52:57,643 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/default/TestLogRolling-testSlowSyncLogRolling/ffe6ec725dba0aad1ddfbd143a15baa0/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-23T22:52:57,643 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-23T22:52:57,644 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-23T22:52:57,645 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:57,645 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffe6ec725dba0aad1ddfbd143a15baa0:
Waiting for close lock at 1732402377570Running coprocessor pre-close hooks at 1732402377571 (+1 ms)Disabling compacts and flushes for region at 1732402377571Disabling writes for close at 1732402377572 (+1 ms)Obtaining lock to block concurrent updates at 1732402377572Preparing flush snapshotting stores in ffe6ec725dba0aad1ddfbd143a15baa0 at 1732402377572Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732402377573 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0. at 1732402377575 (+2 ms)Flushing ffe6ec725dba0aad1ddfbd143a15baa0/info: creating writer at 1732402377575Flushing ffe6ec725dba0aad1ddfbd143a15baa0/info: appending metadata at 1732402377579 (+4 ms)Flushing ffe6ec725dba0aad1ddfbd143a15baa0/info: closing flushed file at 1732402377579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f0a978: reopening flushed file at 1732402377597 (+18 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for ffe6ec725dba0aad1ddfbd143a15baa0 in 37ms, sequenceid=48, compaction requested=true at 1732402377610 (+13 ms)Writing region close event to WAL at 1732402377638 (+28 ms)Running coprocessor post-close hooks at 1732402377644 (+6 ms)Closed at 1732402377645 (+1 ms)
2024-11-23T22:52:57,646 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732402284734.ffe6ec725dba0aad1ddfbd143a15baa0.
2024-11-23T22:52:57,649 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:52:57,665 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/table/e6ae2b485c0b486eabb1841a0388864e is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732402285600/Put/seqid=0
2024-11-23T22:52:57,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741853_1029 (size=5396)
2024-11-23T22:52:57,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741853_1029 (size=5396)
2024-11-23T22:52:57,672 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/table/e6ae2b485c0b486eabb1841a0388864e
2024-11-23T22:52:57,680 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/info/6e8131678a2f4a938c558072e89ca80f as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/info/6e8131678a2f4a938c558072e89ca80f
2024-11-23T22:52:57,689 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/info/6e8131678a2f4a938c558072e89ca80f, entries=10, sequenceid=11, filesize=6.9 K
2024-11-23T22:52:57,691 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/ns/72925dddfdbe4df08a21f040ce4ccc66 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/ns/72925dddfdbe4df08a21f040ce4ccc66
2024-11-23T22:52:57,699 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/ns/72925dddfdbe4df08a21f040ce4ccc66, entries=2, sequenceid=11, filesize=5.0 K
2024-11-23T22:52:57,701 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/.tmp/table/e6ae2b485c0b486eabb1841a0388864e as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/table/e6ae2b485c0b486eabb1841a0388864e
2024-11-23T22:52:57,709 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/table/e6ae2b485c0b486eabb1841a0388864e, entries=2, sequenceid=11, filesize=5.3 K
2024-11-23T22:52:57,710 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 137ms, sequenceid=11, compaction requested=false
2024-11-23T22:52:57,715 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-23T22:52:57,716 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:52:57,716 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:52:57,717 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402377572Running coprocessor pre-close hooks at 1732402377572Disabling compacts and flushes for region at 1732402377573 (+1 ms)Disabling writes for close at 1732402377573Obtaining lock to block concurrent updates at 1732402377574 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732402377574Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732402377574Flushing stores of hbase:meta,,1.1588230740 at 1732402377575 (+1 ms)Flushing 1588230740/info: creating writer at 1732402377575Flushing 1588230740/info: appending metadata at 1732402377596 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732402377596Flushing 1588230740/ns: creating writer at 1732402377612 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732402377629 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732402377629Flushing 1588230740/table: creating writer at 1732402377646 (+17 ms)Flushing 1588230740/table: appending metadata at 1732402377665 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732402377665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498c4b85: reopening flushed file at 1732402377679 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5faa5162: reopening flushed file at 1732402377690 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29f82235: reopening flushed file at 1732402377699 (+9 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 137ms, sequenceid=11, compaction requested=false at 1732402377710 (+11 ms)Writing region close event to WAL at 1732402377711 (+1 ms)Running coprocessor post-close hooks at 1732402377716 (+5 ms)Closed at 1732402377716
2024-11-23T22:52:57,717 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:52:57,774 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,38141,1732402282326; all regions closed.
2024-11-23T22:52:57,777 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,777 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,777 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,778 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,778 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741834_1010 (size=3066)
2024-11-23T22:52:57,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741834_1010 (size=3066)
2024-11-23T22:52:57,788 DEBUG [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs
2024-11-23T22:52:57,788 INFO  [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C38141%2C1732402282326.meta:.meta(num 1732402284222)
2024-11-23T22:52:57,789 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,789 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,789 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,789 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,789 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:57,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741847_1023 (size=12695)
2024-11-23T22:52:57,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741847_1023 (size=12695)
2024-11-23T22:52:57,795 DEBUG [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/oldWALs
2024-11-23T22:52:57,795 INFO  [RS:0;dd96e8dc8369:38141 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C38141%2C1732402282326:(num 1732402357421)
2024-11-23T22:52:57,795 DEBUG [RS:0;dd96e8dc8369:38141 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:52:57,795 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:52:57,796 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:52:57,796 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:52:57,796 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:52:57,796 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:52:57,797 INFO  [RS:0;dd96e8dc8369:38141 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38141
2024-11-23T22:52:57,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,38141,1732402282326
2024-11-23T22:52:57,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:52:57,805 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:52:57,816 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,38141,1732402282326]
2024-11-23T22:52:57,827 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,38141,1732402282326 already deleted, retry=false
2024-11-23T22:52:57,827 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,38141,1732402282326 expired; onlineServers=0
2024-11-23T22:52:57,827 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,33503,1732402281583' *****
2024-11-23T22:52:57,827 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:52:57,827 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:52:57,827 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:52:57,828 DEBUG [M:0;dd96e8dc8369:33503 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:52:57,828 DEBUG [M:0;dd96e8dc8369:33503 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:52:57,828 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:52:57,828 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402283457 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402283457,5,FailOnTimeoutGroup]
2024-11-23T22:52:57,828 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402283457 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402283457,5,FailOnTimeoutGroup]
2024-11-23T22:52:57,828 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:52:57,829 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:52:57,829 DEBUG [M:0;dd96e8dc8369:33503 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:52:57,829 INFO  [M:0;dd96e8dc8369:33503 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:52:57,829 INFO  [M:0;dd96e8dc8369:33503 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:52:57,830 INFO  [M:0;dd96e8dc8369:33503 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:52:57,830 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:52:57,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:52:57,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:52:57,838 DEBUG [M:0;dd96e8dc8369:33503 {}] zookeeper.ZKUtil(347): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:52:57,838 WARN  [M:0;dd96e8dc8369:33503 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:52:57,839 INFO  [M:0;dd96e8dc8369:33503 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/.lastflushedseqids
2024-11-23T22:52:57,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741854_1030 (size=130)
2024-11-23T22:52:57,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741854_1030 (size=130)
2024-11-23T22:52:57,851 INFO  [M:0;dd96e8dc8369:33503 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:52:57,851 INFO  [M:0;dd96e8dc8369:33503 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:52:57,851 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:52:57,851 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:52:57,851 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:52:57,851 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:52:57,851 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:52:57,852 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23 KB heapSize=29.17 KB
2024-11-23T22:52:57,870 DEBUG [M:0;dd96e8dc8369:33503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0cb9f221257a4a8cbe96b539be5cd70e is 82, key is hbase:meta,,1/info:regioninfo/1732402284304/Put/seqid=0
2024-11-23T22:52:57,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741855_1031 (size=5672)
2024-11-23T22:52:57,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741855_1031 (size=5672)
2024-11-23T22:52:57,876 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0cb9f221257a4a8cbe96b539be5cd70e
2024-11-23T22:52:57,899 DEBUG [M:0;dd96e8dc8369:33503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf42f50ff704216be613ed6d1793258 is 764, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402285609/Put/seqid=0
2024-11-23T22:52:57,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741856_1032 (size=6245)
2024-11-23T22:52:57,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741856_1032 (size=6245)
2024-11-23T22:52:57,907 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.40 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf42f50ff704216be613ed6d1793258
2024-11-23T22:52:57,914 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf42f50ff704216be613ed6d1793258
2024-11-23T22:52:57,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:52:57,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38141-0x10169e4f4f30001, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:52:57,917 INFO  [RS:0;dd96e8dc8369:38141 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:52:57,917 INFO  [RS:0;dd96e8dc8369:38141 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,38141,1732402282326; zookeeper connection closed.
2024-11-23T22:52:57,918 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@412c7cdf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@412c7cdf
2024-11-23T22:52:57,918 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:52:57,932 DEBUG [M:0;dd96e8dc8369:33503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60abe424e98c4c2490e2ad56e14a23dd is 69, key is dd96e8dc8369,38141,1732402282326/rs:state/1732402283561/Put/seqid=0
2024-11-23T22:52:57,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741857_1033 (size=5156)
2024-11-23T22:52:57,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741857_1033 (size=5156)
2024-11-23T22:52:57,938 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60abe424e98c4c2490e2ad56e14a23dd
2024-11-23T22:52:57,961 DEBUG [M:0;dd96e8dc8369:33503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/39120cf794574d1a972e5a4e1f7a8e8c is 52, key is load_balancer_on/state:d/1732402284715/Put/seqid=0
2024-11-23T22:52:57,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741858_1034 (size=5056)
2024-11-23T22:52:57,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741858_1034 (size=5056)
2024-11-23T22:52:57,967 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/39120cf794574d1a972e5a4e1f7a8e8c
2024-11-23T22:52:57,975 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0cb9f221257a4a8cbe96b539be5cd70e as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0cb9f221257a4a8cbe96b539be5cd70e
2024-11-23T22:52:57,982 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0cb9f221257a4a8cbe96b539be5cd70e, entries=8, sequenceid=59, filesize=5.5 K
2024-11-23T22:52:57,983 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf42f50ff704216be613ed6d1793258 as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf42f50ff704216be613ed6d1793258
2024-11-23T22:52:57,989 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf42f50ff704216be613ed6d1793258
2024-11-23T22:52:57,989 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf42f50ff704216be613ed6d1793258, entries=6, sequenceid=59, filesize=6.1 K
2024-11-23T22:52:57,991 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/60abe424e98c4c2490e2ad56e14a23dd as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60abe424e98c4c2490e2ad56e14a23dd
2024-11-23T22:52:57,997 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/60abe424e98c4c2490e2ad56e14a23dd, entries=1, sequenceid=59, filesize=5.0 K
2024-11-23T22:52:57,999 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/39120cf794574d1a972e5a4e1f7a8e8c as hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/39120cf794574d1a972e5a4e1f7a8e8c
2024-11-23T22:52:58,005 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/39120cf794574d1a972e5a4e1f7a8e8c, entries=1, sequenceid=59, filesize=4.9 K
2024-11-23T22:52:58,006 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23 KB/23552, heapSize ~29.11 KB/29808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 155ms, sequenceid=59, compaction requested=false
2024-11-23T22:52:58,008 INFO  [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:52:58,008 DEBUG [M:0;dd96e8dc8369:33503 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402377851Disabling compacts and flushes for region at 1732402377851Disabling writes for close at 1732402377851Obtaining lock to block concurrent updates at 1732402377852 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402377852Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23552, getHeapSize=29808, getOffHeapSize=0, getCellsCount=70 at 1732402377852Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402377853 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402377853Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402377870 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402377870Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402377883 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402377898 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402377898Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402377915 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402377931 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402377931Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402377945 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402377960 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402377960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ebf01d: reopening flushed file at 1732402377973 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bc60b44: reopening flushed file at 1732402377982 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5de72ee5: reopening flushed file at 1732402377989 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b504463: reopening flushed file at 1732402377997 (+8 ms)Finished flush of dataSize ~23 KB/23552, heapSize ~29.11 KB/29808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 155ms, sequenceid=59, compaction requested=false at 1732402378006 (+9 ms)Writing region close event to WAL at 1732402378008 (+2 ms)Closed at 1732402378008
2024-11-23T22:52:58,009 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:58,009 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:58,009 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:58,009 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:58,009 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:52:58,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40255 is added to blk_1073741830_1006 (size=27949)
2024-11-23T22:52:58,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38417 is added to blk_1073741830_1006 (size=27949)
2024-11-23T22:52:58,012 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:52:58,012 INFO  [M:0;dd96e8dc8369:33503 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:52:58,013 INFO  [M:0;dd96e8dc8369:33503 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33503
2024-11-23T22:52:58,013 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:52:58,127 INFO  [M:0;dd96e8dc8369:33503 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:52:58,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:52:58,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33503-0x10169e4f4f30000, quorum=127.0.0.1:49185, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:52:58,132 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@590b36b7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:52:58,134 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@237fc06a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:52:58,134 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:52:58,135 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35f1cf70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:52:58,135 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43794ae7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,STOPPED}
2024-11-23T22:52:58,138 WARN  [BP-294770882-172.17.0.2-1732402276733 heartbeating to localhost/127.0.0.1:35437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:52:58,138 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:52:58,138 WARN  [BP-294770882-172.17.0.2-1732402276733 heartbeating to localhost/127.0.0.1:35437 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-294770882-172.17.0.2-1732402276733 (Datanode Uuid c2f8c6ce-6be1-4e37-806a-6817b37d2694) service to localhost/127.0.0.1:35437
2024-11-23T22:52:58,138 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:52:58,139 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data3/current/BP-294770882-172.17.0.2-1732402276733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:52:58,139 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data4/current/BP-294770882-172.17.0.2-1732402276733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:52:58,140 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:52:58,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11e88411{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:52:58,146 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75423500{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:52:58,146 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:52:58,146 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@621a7cbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:52:58,146 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26c88bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,STOPPED}
2024-11-23T22:52:58,148 WARN  [BP-294770882-172.17.0.2-1732402276733 heartbeating to localhost/127.0.0.1:35437 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:52:58,148 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:52:58,148 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:52:58,148 WARN  [BP-294770882-172.17.0.2-1732402276733 heartbeating to localhost/127.0.0.1:35437 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-294770882-172.17.0.2-1732402276733 (Datanode Uuid 5a84c4f6-973d-43d9-be1f-fc255b5cbf8e) service to localhost/127.0.0.1:35437
2024-11-23T22:52:58,148 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data1/current/BP-294770882-172.17.0.2-1732402276733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:52:58,149 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/cluster_890f3df6-d985-af71-06bb-c9ccefa37ab5/data/data2/current/BP-294770882-172.17.0.2-1732402276733 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:52:58,149 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:52:58,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de997b9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:52:58,158 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:52:58,158 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:52:58,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:52:58,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir/,STOPPED}
2024-11-23T22:52:58,166 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:52:58,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:52:58,206 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=81 (was 12)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35437
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:35437 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35437
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1ab9ef81
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/dd96e8dc8369:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35437
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35437
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35437
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/dd96e8dc8369:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/dd96e8dc8369:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: ForkJoinPool-2-worker-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:35437 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:35437 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 294), ProcessCount=11 (was 11), AvailableMemoryMB=7876 (was 7687) - AvailableMemoryMB LEAK? -
2024-11-23T22:52:58,213 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=82, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=7875
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.log.dir so I do NOT create it in target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0742054-6ef9-7a6c-23ef-ab8a7f062d17/hadoop.tmp.dir so I do NOT create it in target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631, deleteOnExit=true
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:52:58,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/test.cache.data in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:52:58,215 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:52:58,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:52:58,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:52:58,230 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:52:58,513 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:52:58,520 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:52:58,524 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:52:58,524 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:52:58,524 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:52:58,525 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:52:58,526 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:52:58,526 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:52:58,622 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2787d4b6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/java.io.tmpdir/jetty-localhost-32775-hadoop-hdfs-3_4_1-tests_jar-_-any-12785408793752855931/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:52:58,623 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:32775}
2024-11-23T22:52:58,623 INFO  [Time-limited test {}] server.Server(415): Started @103948ms
2024-11-23T22:52:58,636 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:52:58,950 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:52:58,956 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:52:58,957 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:52:58,957 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:52:58,957 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:52:58,958 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f754f75{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:52:58,958 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a3a779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:52:59,056 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12742a74{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/java.io.tmpdir/jetty-localhost-41245-hadoop-hdfs-3_4_1-tests_jar-_-any-2431250586452910281/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:52:59,056 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f1e2b4c{HTTP/1.1, (http/1.1)}{localhost:41245}
2024-11-23T22:52:59,056 INFO  [Time-limited test {}] server.Server(415): Started @104381ms
2024-11-23T22:52:59,058 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:52:59,093 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:52:59,097 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:52:59,098 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:52:59,098 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:52:59,098 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:52:59,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@592c88e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:52:59,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c5ed954{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:52:59,196 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62c9cc57{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/java.io.tmpdir/jetty-localhost-33841-hadoop-hdfs-3_4_1-tests_jar-_-any-6782768562722736538/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:52:59,196 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3756c399{HTTP/1.1, (http/1.1)}{localhost:33841}
2024-11-23T22:52:59,196 INFO  [Time-limited test {}] server.Server(415): Started @104521ms
2024-11-23T22:52:59,199 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:00,193 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data1/current/BP-1870164089-172.17.0.2-1732402378241/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:00,193 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data2/current/BP-1870164089-172.17.0.2-1732402378241/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:00,215 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:00,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b31e5f3332e4f1c with lease ID 0x8c542d3282d03419: Processing first storage report for DS-86a96f68-5dd8-467a-be1a-51f5d0f278af from datanode DatanodeRegistration(127.0.0.1:40221, datanodeUuid=c4fb8e0e-91af-4ca6-bd95-c430f2953387, infoPort=37733, infoSecurePort=0, ipcPort=44027, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241)
2024-11-23T22:53:00,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b31e5f3332e4f1c with lease ID 0x8c542d3282d03419: from storage DS-86a96f68-5dd8-467a-be1a-51f5d0f278af node DatanodeRegistration(127.0.0.1:40221, datanodeUuid=c4fb8e0e-91af-4ca6-bd95-c430f2953387, infoPort=37733, infoSecurePort=0, ipcPort=44027, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:00,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b31e5f3332e4f1c with lease ID 0x8c542d3282d03419: Processing first storage report for DS-5294437b-5485-4b23-8fba-d87e7e0d847f from datanode DatanodeRegistration(127.0.0.1:40221, datanodeUuid=c4fb8e0e-91af-4ca6-bd95-c430f2953387, infoPort=37733, infoSecurePort=0, ipcPort=44027, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241)
2024-11-23T22:53:00,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b31e5f3332e4f1c with lease ID 0x8c542d3282d03419: from storage DS-5294437b-5485-4b23-8fba-d87e7e0d847f node DatanodeRegistration(127.0.0.1:40221, datanodeUuid=c4fb8e0e-91af-4ca6-bd95-c430f2953387, infoPort=37733, infoSecurePort=0, ipcPort=44027, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-23T22:53:00,312 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data3/current/BP-1870164089-172.17.0.2-1732402378241/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:00,312 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data4/current/BP-1870164089-172.17.0.2-1732402378241/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:00,331 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:00,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef8ed750480ae412 with lease ID 0x8c542d3282d0341a: Processing first storage report for DS-61b062d2-a2a1-4ad5-9422-c7187a694f0a from datanode DatanodeRegistration(127.0.0.1:37621, datanodeUuid=3454a398-28b0-4859-aeee-0fa282cfb16b, infoPort=33959, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241)
2024-11-23T22:53:00,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef8ed750480ae412 with lease ID 0x8c542d3282d0341a: from storage DS-61b062d2-a2a1-4ad5-9422-c7187a694f0a node DatanodeRegistration(127.0.0.1:37621, datanodeUuid=3454a398-28b0-4859-aeee-0fa282cfb16b, infoPort=33959, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:00,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef8ed750480ae412 with lease ID 0x8c542d3282d0341a: Processing first storage report for DS-591b7a94-bc89-4d9d-91e9-08e0501bcbe7 from datanode DatanodeRegistration(127.0.0.1:37621, datanodeUuid=3454a398-28b0-4859-aeee-0fa282cfb16b, infoPort=33959, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241)
2024-11-23T22:53:00,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef8ed750480ae412 with lease ID 0x8c542d3282d0341a: from storage DS-591b7a94-bc89-4d9d-91e9-08e0501bcbe7 node DatanodeRegistration(127.0.0.1:37621, datanodeUuid=3454a398-28b0-4859-aeee-0fa282cfb16b, infoPort=33959, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=381408657;c=1732402378241), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:00,336 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e
2024-11-23T22:53:00,339 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/zookeeper_0, clientPort=51258, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:53:00,341 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51258
2024-11-23T22:53:00,341 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,342 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:00,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:00,356 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066 with version=8
2024-11-23T22:53:00,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:53:00,358 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:00,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,359 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:00,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,359 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:00,359 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:53:00,359 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:00,360 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37739
2024-11-23T22:53:00,361 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37739 connecting to ZooKeeper ensemble=127.0.0.1:51258
2024-11-23T22:53:00,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377390x0, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:00,411 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37739-0x10169e679de0000 connected
2024-11-23T22:53:00,495 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,499 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:00,504 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066, hbase.cluster.distributed=false
2024-11-23T22:53:00,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:00,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37739
2024-11-23T22:53:00,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37739
2024-11-23T22:53:00,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37739
2024-11-23T22:53:00,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37739
2024-11-23T22:53:00,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37739
2024-11-23T22:53:00,529 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:00,530 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:53:00,531 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:00,531 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43667
2024-11-23T22:53:00,534 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43667 connecting to ZooKeeper ensemble=127.0.0.1:51258
2024-11-23T22:53:00,535 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,538 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,547 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436670x0, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:00,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436670x0, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:00,549 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:53:00,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43667-0x10169e679de0001 connected
2024-11-23T22:53:00,561 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:53:00,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:53:00,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:00,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43667
2024-11-23T22:53:00,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43667
2024-11-23T22:53:00,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43667
2024-11-23T22:53:00,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43667
2024-11-23T22:53:00,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43667
2024-11-23T22:53:00,587 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:37739
2024-11-23T22:53:00,588 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:00,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:00,599 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:00,601 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:00,610 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:53:00,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:00,610 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:00,612 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:53:00,612 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,37739,1732402380358 from backup master directory
2024-11-23T22:53:00,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:00,620 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:00,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:00,621 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:00,621 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:00,630 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/hbase.id] with ID: 9db37797-63e0-4902-b1b4-04c0488f846a
2024-11-23T22:53:00,630 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/.tmp/hbase.id
2024-11-23T22:53:00,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:00,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:00,652 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/.tmp/hbase.id]:[hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/hbase.id]
2024-11-23T22:53:00,669 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:00,669 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:53:00,672 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-23T22:53:00,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:00,684 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:00,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:00,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:01,094 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:53:01,096 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:53:01,096 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:01,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:01,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:01,110 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store
2024-11-23T22:53:01,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:01,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:01,119 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:01,120 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:53:01,120 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:01,120 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:01,120 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:53:01,120 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:01,120 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:01,120 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402381120Disabling compacts and flushes for region at 1732402381120Disabling writes for close at 1732402381120Writing region close event to WAL at 1732402381120Closed at 1732402381120
2024-11-23T22:53:01,122 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/.initializing
2024-11-23T22:53:01,122 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/WALs/dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:01,126 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C37739%2C1732402380358, suffix=, logDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/WALs/dd96e8dc8369,37739,1732402380358, archiveDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/oldWALs, maxLogs=10
2024-11-23T22:53:01,127 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C37739%2C1732402380358.1732402381126
2024-11-23T22:53:01,138 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/WALs/dd96e8dc8369,37739,1732402380358/dd96e8dc8369%2C37739%2C1732402380358.1732402381126
2024-11-23T22:53:01,139 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37733:37733),(127.0.0.1/127.0.0.1:33959:33959)]
2024-11-23T22:53:01,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:01,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:01,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,143 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:01,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:53:01,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:01,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:01,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:53:01,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:01,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:01,152 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:53:01,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,153 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:01,153 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:01,155 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:53:01,155 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,156 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:01,156 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,157 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,157 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:53:01,161 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:01,163 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:01,164 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751536, jitterRate=-0.0443735271692276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:53:01,165 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402381141Initializing all the Stores at 1732402381142 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402381142Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402381143 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402381143Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402381143Cleaning up temporary data from old regions at 1732402381159 (+16 ms)Region opened successfully at 1732402381165 (+6 ms)
2024-11-23T22:53:01,165 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:53:01,169 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c4745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:01,170 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:53:01,171 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:53:01,171 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:53:01,171 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:53:01,171 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:53:01,172 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:53:01,172 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:53:01,174 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:53:01,175 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:53:01,217 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:53:01,218 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:53:01,219 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:53:01,231 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:53:01,232 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:53:01,234 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:53:01,241 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:53:01,243 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:53:01,252 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:53:01,257 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:53:01,270 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:53:01,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:01,284 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:01,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,284 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,285 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,37739,1732402380358, sessionid=0x10169e679de0000, setting cluster-up flag (Was=false)
2024-11-23T22:53:01,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,304 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,336 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:53:01,339 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:01,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,358 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,389 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:53:01,394 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:01,397 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:53:01,400 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:01,401 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:53:01,401 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:53:01,401 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,37739,1732402380358
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:53:01,405 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:01,405 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:01,405 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:01,405 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:01,406 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:53:01,406 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,406 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:01,406 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,408 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402411407
2024-11-23T22:53:01,408 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:53:01,408 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:53:01,408 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:53:01,408 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:53:01,409 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:53:01,409 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:53:01,409 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,410 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:01,410 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:53:01,410 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:53:01,410 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:53:01,410 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:53:01,411 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:53:01,411 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:53:01,412 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402381411,5,FailOnTimeoutGroup]
2024-11-23T22:53:01,412 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402381412,5,FailOnTimeoutGroup]
2024-11-23T22:53:01,412 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,412 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:53:01,412 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,412 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,412 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,412 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:53:01,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:01,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:01,423 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:53:01,423 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066
2024-11-23T22:53:01,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:01,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:01,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:01,433 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:01,435 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:01,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:01,436 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:01,437 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:01,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:01,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:01,440 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:01,440 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,441 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:01,441 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:01,443 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:01,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:01,443 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:01,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:01,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740
2024-11-23T22:53:01,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740
2024-11-23T22:53:01,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:01,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:01,447 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:01,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:01,451 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:01,451 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817663, jitterRate=0.03971315920352936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:01,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402381431Initializing all the Stores at 1732402381432 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402381432Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402381432Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402381432Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402381432Cleaning up temporary data from old regions at 1732402381446 (+14 ms)Region opened successfully at 1732402381452 (+6 ms)
2024-11-23T22:53:01,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:53:01,453 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:53:01,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:53:01,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:53:01,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:53:01,453 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:01,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402381452Disabling compacts and flushes for region at 1732402381452Disabling writes for close at 1732402381453 (+1 ms)Writing region close event to WAL at 1732402381453Closed at 1732402381453
2024-11-23T22:53:01,455 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:01,455 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:53:01,455 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:53:01,456 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:01,458 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:53:01,475 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(746): ClusterId : 9db37797-63e0-4902-b1b4-04c0488f846a
2024-11-23T22:53:01,475 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:53:01,485 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:53:01,485 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:53:01,495 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:53:01,495 DEBUG [RS:0;dd96e8dc8369:43667 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a96a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:01,506 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:43667
2024-11-23T22:53:01,507 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:53:01,507 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:53:01,507 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:53:01,508 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,37739,1732402380358 with port=43667, startcode=1732402380529
2024-11-23T22:53:01,508 DEBUG [RS:0;dd96e8dc8369:43667 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:53:01,510 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38977, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:53:01,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37739 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37739 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,514 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066
2024-11-23T22:53:01,514 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35407
2024-11-23T22:53:01,514 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:53:01,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:01,526 DEBUG [RS:0;dd96e8dc8369:43667 {}] zookeeper.ZKUtil(111): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,527 WARN  [RS:0;dd96e8dc8369:43667 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:01,527 INFO  [RS:0;dd96e8dc8369:43667 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:01,527 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/WALs/dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,527 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,43667,1732402380529]
2024-11-23T22:53:01,532 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:53:01,534 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:53:01,535 INFO  [RS:0;dd96e8dc8369:43667 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:53:01,535 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,535 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:53:01,536 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:53:01,536 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,536 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,536 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:01,537 DEBUG [RS:0;dd96e8dc8369:43667 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,539 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43667,1732402380529-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:01,557 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:53:01,558 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43667,1732402380529-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,558 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,558 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.Replication(171): dd96e8dc8369,43667,1732402380529 started
2024-11-23T22:53:01,575 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:01,576 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,43667,1732402380529, RpcServer on dd96e8dc8369/172.17.0.2:43667, sessionid=0x10169e679de0001
2024-11-23T22:53:01,576 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:53:01,576 DEBUG [RS:0;dd96e8dc8369:43667 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,576 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43667,1732402380529'
2024-11-23T22:53:01,576 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:53:01,577 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43667,1732402380529'
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:53:01,578 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:53:01,579 DEBUG [RS:0;dd96e8dc8369:43667 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:53:01,579 INFO  [RS:0;dd96e8dc8369:43667 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:53:01,579 INFO  [RS:0;dd96e8dc8369:43667 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:53:01,608 WARN  [dd96e8dc8369:37739 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:53:01,681 INFO  [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C43667%2C1732402380529, suffix=, logDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/WALs/dd96e8dc8369,43667,1732402380529, archiveDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/oldWALs, maxLogs=32
2024-11-23T22:53:01,683 INFO  [RS:0;dd96e8dc8369:43667 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43667%2C1732402380529.1732402381683
2024-11-23T22:53:01,690 INFO  [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/WALs/dd96e8dc8369,43667,1732402380529/dd96e8dc8369%2C43667%2C1732402380529.1732402381683
2024-11-23T22:53:01,692 DEBUG [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33959:33959),(127.0.0.1/127.0.0.1:37733:37733)]
2024-11-23T22:53:01,858 DEBUG [dd96e8dc8369:37739 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:53:01,859 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:01,861 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,43667,1732402380529, state=OPENING
2024-11-23T22:53:01,912 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:53:01,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:53:01,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:53:01,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-23T22:53:01,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,922 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:01,923 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:01,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:01,923 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43667,1732402380529}]
2024-11-23T22:53:01,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:02,078 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:53:02,081 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:53:02,087 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:53:02,088 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:02,092 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C43667%2C1732402380529.meta, suffix=.meta, logDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/WALs/dd96e8dc8369,43667,1732402380529, archiveDir=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/oldWALs, maxLogs=32
2024-11-23T22:53:02,095 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43667%2C1732402380529.meta.1732402382094.meta
2024-11-23T22:53:02,107 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/WALs/dd96e8dc8369,43667,1732402380529/dd96e8dc8369%2C43667%2C1732402380529.meta.1732402382094.meta
2024-11-23T22:53:02,114 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33959:33959),(127.0.0.1/127.0.0.1:37733:37733)]
2024-11-23T22:53:02,121 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:53:02,122 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:53:02,122 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:53:02,125 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:02,129 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:02,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:02,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:02,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:02,132 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:02,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:02,133 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:02,133 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:02,134 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:02,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:02,135 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:02,135 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:02,137 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:02,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:02,137 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:02,138 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:02,139 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740
2024-11-23T22:53:02,141 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740
2024-11-23T22:53:02,142 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:02,142 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:02,143 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:02,145 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:02,146 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=867054, jitterRate=0.102516770362854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:02,146 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:53:02,147 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402382123Writing region info on filesystem at 1732402382123Initializing all the Stores at 1732402382124 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402382124Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402382125 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402382125Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402382125Cleaning up temporary data from old regions at 1732402382142 (+17 ms)Running coprocessor post-open hooks at 1732402382146 (+4 ms)Region opened successfully at 1732402382147 (+1 ms)
2024-11-23T22:53:02,149 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402382077
2024-11-23T22:53:02,153 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:53:02,153 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:53:02,154 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:02,156 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,43667,1732402380529, state=OPEN
2024-11-23T22:53:02,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:53:02,208 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:53:02,208 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:02,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:02,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:02,213 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:53:02,213 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43667,1732402380529 in 285 msec
2024-11-23T22:53:02,216 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:53:02,216 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 758 msec
2024-11-23T22:53:02,218 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:02,218 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:53:02,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:53:02,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,43667,1732402380529, seqNum=-1]
2024-11-23T22:53:02,220 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:53:02,222 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37209, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:53:02,229 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 830 msec
2024-11-23T22:53:02,230 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402382230, completionTime=-1
2024-11-23T22:53:02,230 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:53:02,230 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:53:02,232 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:53:02,232 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402442232
2024-11-23T22:53:02,232 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402502232
2024-11-23T22:53:02,232 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:37739, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,233 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,236 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.619sec
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:02,240 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:53:02,243 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:53:02,244 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:53:02,244 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37739,1732402380358-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:02,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2f75ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:02,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,37739,-1 for getting cluster id
2024-11-23T22:53:02,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:53:02,277 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9db37797-63e0-4902-b1b4-04c0488f846a'
2024-11-23T22:53:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:53:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9db37797-63e0-4902-b1b4-04c0488f846a"

2024-11-23T22:53:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36139299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,37739,-1]
2024-11-23T22:53:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:53:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,281 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:53:02,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d78aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:02,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:53:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,43667,1732402380529, seqNum=-1]
2024-11-23T22:53:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:53:02,285 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:53:02,287 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:02,288 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:02,291 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:53:02,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:53:02,291 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:53:02,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:02,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,291 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:53:02,292 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:53:02,292 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1676223287, stopped=false
2024-11-23T22:53:02,292 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,37739,1732402380358
2024-11-23T22:53:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:02,315 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:02,315 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:02,315 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:53:02,316 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:53:02,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:02,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:02,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:02,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,316 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,43667,1732402380529' *****
2024-11-23T22:53:02,316 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:53:02,317 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:43667.
2024-11-23T22:53:02,317 DEBUG [RS:0;dd96e8dc8369:43667 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:02,317 DEBUG [RS:0;dd96e8dc8369:43667 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:53:02,317 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:53:02,318 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-23T22:53:02,318 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-23T22:53:02,318 DEBUG [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-23T22:53:02,318 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:53:02,318 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:53:02,318 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:53:02,318 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:53:02,318 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:53:02,318 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-23T22:53:02,335 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/.tmp/ns/0ebb4f9f3f1d4edcaffe2eb2cbabe372 is 43, key is default/ns:d/1732402382223/Put/seqid=0
2024-11-23T22:53:02,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741835_1011 (size=5153)
2024-11-23T22:53:02,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741835_1011 (size=5153)
2024-11-23T22:53:02,341 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/.tmp/ns/0ebb4f9f3f1d4edcaffe2eb2cbabe372
2024-11-23T22:53:02,349 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/.tmp/ns/0ebb4f9f3f1d4edcaffe2eb2cbabe372 as hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/ns/0ebb4f9f3f1d4edcaffe2eb2cbabe372
2024-11-23T22:53:02,357 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/ns/0ebb4f9f3f1d4edcaffe2eb2cbabe372, entries=2, sequenceid=6, filesize=5.0 K
2024-11-23T22:53:02,359 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false
2024-11-23T22:53:02,359 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:53:02,365 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-23T22:53:02,365 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:53:02,365 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:02,366 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402382318Running coprocessor pre-close hooks at 1732402382318Disabling compacts and flushes for region at 1732402382318Disabling writes for close at 1732402382318Obtaining lock to block concurrent updates at 1732402382318Preparing flush snapshotting stores in 1588230740 at 1732402382318Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732402382319 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732402382320 (+1 ms)Flushing 1588230740/ns: creating writer at 1732402382320Flushing 1588230740/ns: appending metadata at 1732402382334 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732402382334Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cc61758: reopening flushed file at 1732402382348 (+14 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false at 1732402382359 (+11 ms)Writing region close event to WAL at 1732402382360 (+1 ms)Running coprocessor post-close hooks at 1732402382365 (+5 ms)Closed at 1732402382365
2024-11-23T22:53:02,366 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:02,518 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,43667,1732402380529; all regions closed.
2024-11-23T22:53:02,519 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,520 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,520 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,521 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,521 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741834_1010 (size=1152)
2024-11-23T22:53:02,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741834_1010 (size=1152)
2024-11-23T22:53:02,531 DEBUG [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/oldWALs
2024-11-23T22:53:02,531 INFO  [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C43667%2C1732402380529.meta:.meta(num 1732402382094)
2024-11-23T22:53:02,532 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,532 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,532 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,533 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,533 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741833_1009 (size=93)
2024-11-23T22:53:02,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741833_1009 (size=93)
2024-11-23T22:53:02,540 DEBUG [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/oldWALs
2024-11-23T22:53:02,540 INFO  [RS:0;dd96e8dc8369:43667 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C43667%2C1732402380529:(num 1732402381683)
2024-11-23T22:53:02,540 DEBUG [RS:0;dd96e8dc8369:43667 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:02,540 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:02,540 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:53:02,540 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:53:02,541 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:53:02,541 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:53:02,541 INFO  [RS:0;dd96e8dc8369:43667 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43667
2024-11-23T22:53:02,554 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,43667,1732402380529
2024-11-23T22:53:02,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:02,554 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:53:02,565 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,43667,1732402380529]
2024-11-23T22:53:02,575 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,43667,1732402380529 already deleted, retry=false
2024-11-23T22:53:02,575 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,43667,1732402380529 expired; onlineServers=0
2024-11-23T22:53:02,575 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,37739,1732402380358' *****
2024-11-23T22:53:02,575 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:53:02,576 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:53:02,576 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:53:02,576 DEBUG [M:0;dd96e8dc8369:37739 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:53:02,576 DEBUG [M:0;dd96e8dc8369:37739 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:53:02,576 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:53:02,576 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402381412 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402381412,5,FailOnTimeoutGroup]
2024-11-23T22:53:02,576 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402381411 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402381411,5,FailOnTimeoutGroup]
2024-11-23T22:53:02,577 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:53:02,577 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:53:02,577 DEBUG [M:0;dd96e8dc8369:37739 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:53:02,578 INFO  [M:0;dd96e8dc8369:37739 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:53:02,578 INFO  [M:0;dd96e8dc8369:37739 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:53:02,578 INFO  [M:0;dd96e8dc8369:37739 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:53:02,578 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:53:02,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:53:02,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:02,589 DEBUG [M:0;dd96e8dc8369:37739 {}] zookeeper.ZKUtil(347): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:53:02,589 WARN  [M:0;dd96e8dc8369:37739 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:53:02,589 INFO  [M:0;dd96e8dc8369:37739 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/.lastflushedseqids
2024-11-23T22:53:02,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741836_1012 (size=99)
2024-11-23T22:53:02,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741836_1012 (size=99)
2024-11-23T22:53:02,597 INFO  [M:0;dd96e8dc8369:37739 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:53:02,597 INFO  [M:0;dd96e8dc8369:37739 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:53:02,597 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:53:02,598 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:02,598 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:02,598 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:53:02,598 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:02,598 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-23T22:53:02,619 DEBUG [M:0;dd96e8dc8369:37739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b064c2f90a254b4eb4865d812e8ec2c6 is 82, key is hbase:meta,,1/info:regioninfo/1732402382154/Put/seqid=0
2024-11-23T22:53:02,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741837_1013 (size=5672)
2024-11-23T22:53:02,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741837_1013 (size=5672)
2024-11-23T22:53:02,624 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b064c2f90a254b4eb4865d812e8ec2c6
2024-11-23T22:53:02,647 DEBUG [M:0;dd96e8dc8369:37739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3f389497c713408ab89a17364d2f98c0 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732402382228/Put/seqid=0
2024-11-23T22:53:02,647 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:02,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741838_1014 (size=5275)
2024-11-23T22:53:02,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741838_1014 (size=5275)
2024-11-23T22:53:02,653 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3f389497c713408ab89a17364d2f98c0
2024-11-23T22:53:02,655 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:02,665 INFO  [RS:0;dd96e8dc8369:43667 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:53:02,665 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:02,665 INFO  [RS:0;dd96e8dc8369:43667 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,43667,1732402380529; zookeeper connection closed.
2024-11-23T22:53:02,665 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43667-0x10169e679de0001, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:02,665 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c237c41 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c237c41
2024-11-23T22:53:02,665 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:53:02,673 DEBUG [M:0;dd96e8dc8369:37739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da2cf2953414079abc138562cba5702 is 69, key is dd96e8dc8369,43667,1732402380529/rs:state/1732402381512/Put/seqid=0
2024-11-23T22:53:02,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741839_1015 (size=5156)
2024-11-23T22:53:02,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741839_1015 (size=5156)
2024-11-23T22:53:02,679 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da2cf2953414079abc138562cba5702
2024-11-23T22:53:02,700 DEBUG [M:0;dd96e8dc8369:37739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/aa6dbf339e8741fc8af71508ccd10293 is 52, key is load_balancer_on/state:d/1732402382289/Put/seqid=0
2024-11-23T22:53:02,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741840_1016 (size=5056)
2024-11-23T22:53:02,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741840_1016 (size=5056)
2024-11-23T22:53:02,706 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/aa6dbf339e8741fc8af71508ccd10293
2024-11-23T22:53:02,714 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b064c2f90a254b4eb4865d812e8ec2c6 as hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b064c2f90a254b4eb4865d812e8ec2c6
2024-11-23T22:53:02,721 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b064c2f90a254b4eb4865d812e8ec2c6, entries=8, sequenceid=29, filesize=5.5 K
2024-11-23T22:53:02,722 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3f389497c713408ab89a17364d2f98c0 as hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3f389497c713408ab89a17364d2f98c0
2024-11-23T22:53:02,728 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3f389497c713408ab89a17364d2f98c0, entries=3, sequenceid=29, filesize=5.2 K
2024-11-23T22:53:02,730 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da2cf2953414079abc138562cba5702 as hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0da2cf2953414079abc138562cba5702
2024-11-23T22:53:02,737 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0da2cf2953414079abc138562cba5702, entries=1, sequenceid=29, filesize=5.0 K
2024-11-23T22:53:02,738 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/aa6dbf339e8741fc8af71508ccd10293 as hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/aa6dbf339e8741fc8af71508ccd10293
2024-11-23T22:53:02,745 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35407/user/jenkins/test-data/91b0a13d-cd12-222c-dcf9-9e8818475066/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/aa6dbf339e8741fc8af71508ccd10293, entries=1, sequenceid=29, filesize=4.9 K
2024-11-23T22:53:02,747 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=29, compaction requested=false
2024-11-23T22:53:02,748 INFO  [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:02,749 DEBUG [M:0;dd96e8dc8369:37739 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402382597Disabling compacts and flushes for region at 1732402382597Disabling writes for close at 1732402382598 (+1 ms)Obtaining lock to block concurrent updates at 1732402382598Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402382598Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732402382599 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402382600 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402382600Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402382618 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402382618Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402382630 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402382646 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402382646Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402382658 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402382672 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402382673 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402382684 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402382699 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402382700 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bd90874: reopening flushed file at 1732402382713 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7980a81b: reopening flushed file at 1732402382721 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5df5c: reopening flushed file at 1732402382729 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@df71da0: reopening flushed file at 1732402382737 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=29, compaction requested=false at 1732402382747 (+10 ms)Writing region close event to WAL at 1732402382748 (+1 ms)Closed at 1732402382748
2024-11-23T22:53:02,749 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,749 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,750 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,750 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,750 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:02,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37621 is added to blk_1073741830_1006 (size=10311)
2024-11-23T22:53:02,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741830_1006 (size=10311)
2024-11-23T22:53:02,753 INFO  [M:0;dd96e8dc8369:37739 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:53:02,753 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:53:02,753 INFO  [M:0;dd96e8dc8369:37739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37739
2024-11-23T22:53:02,753 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:53:02,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:02,865 INFO  [M:0;dd96e8dc8369:37739 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:53:02,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37739-0x10169e679de0000, quorum=127.0.0.1:51258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:02,873 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62c9cc57{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:02,873 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3756c399{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:02,874 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:02,874 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c5ed954{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:02,874 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@592c88e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:02,876 WARN  [BP-1870164089-172.17.0.2-1732402378241 heartbeating to localhost/127.0.0.1:35407 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:02,876 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:02,876 WARN  [BP-1870164089-172.17.0.2-1732402378241 heartbeating to localhost/127.0.0.1:35407 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1870164089-172.17.0.2-1732402378241 (Datanode Uuid 3454a398-28b0-4859-aeee-0fa282cfb16b) service to localhost/127.0.0.1:35407
2024-11-23T22:53:02,876 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:02,877 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data3/current/BP-1870164089-172.17.0.2-1732402378241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:02,877 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data4/current/BP-1870164089-172.17.0.2-1732402378241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:02,877 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:02,881 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12742a74{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:02,882 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f1e2b4c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:02,882 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:02,882 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a3a779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:02,882 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f754f75{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:02,883 WARN  [BP-1870164089-172.17.0.2-1732402378241 heartbeating to localhost/127.0.0.1:35407 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:02,883 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:02,883 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:02,883 WARN  [BP-1870164089-172.17.0.2-1732402378241 heartbeating to localhost/127.0.0.1:35407 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1870164089-172.17.0.2-1732402378241 (Datanode Uuid c4fb8e0e-91af-4ca6-bd95-c430f2953387) service to localhost/127.0.0.1:35407
2024-11-23T22:53:02,884 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data1/current/BP-1870164089-172.17.0.2-1732402378241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:02,884 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:02,884 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/cluster_bfaf4eb3-a889-4a26-e791-cabe678a5631/data/data2/current/BP-1870164089-172.17.0.2-1732402378241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:02,889 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2787d4b6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:53:02,889 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:02,889 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:02,890 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:02,890 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:02,896 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.log.dir so I do NOT create it in target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8120cfc9-f5d5-1995-88e4-28be6b88a49e/hadoop.tmp.dir so I do NOT create it in target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e, deleteOnExit=true
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/test.cache.data in system properties and HBase conf
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:53:02,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:53:02,913 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:53:02,913 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:53:02,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:53:02,927 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:53:03,171 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:53:03,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:03,194 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:03,196 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:03,197 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:03,286 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:03,291 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:03,293 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:03,293 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:03,293 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:53:03,293 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:03,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41fecd82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:03,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@281d64b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:03,388 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6880627a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-40389-hadoop-hdfs-3_4_1-tests_jar-_-any-6732888100911297544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:53:03,388 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d3a3315{HTTP/1.1, (http/1.1)}{localhost:40389}
2024-11-23T22:53:03,389 INFO  [Time-limited test {}] server.Server(415): Started @108713ms
2024-11-23T22:53:03,401 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:53:03,539 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:03,640 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:03,643 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:03,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:03,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:03,644 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:03,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@596eb72c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:03,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34786aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:03,737 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f0e057c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-45241-hadoop-hdfs-3_4_1-tests_jar-_-any-16412094326754224194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:03,738 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26c6facf{HTTP/1.1, (http/1.1)}{localhost:45241}
2024-11-23T22:53:03,738 INFO  [Time-limited test {}] server.Server(415): Started @109062ms
2024-11-23T22:53:03,739 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:03,767 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:03,771 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:03,771 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:03,771 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:03,772 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:03,772 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36318c9a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:03,772 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acad2a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:03,866 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a02718e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-45125-hadoop-hdfs-3_4_1-tests_jar-_-any-14671569694133513247/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:03,867 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@527a79d3{HTTP/1.1, (http/1.1)}{localhost:45125}
2024-11-23T22:53:03,867 INFO  [Time-limited test {}] server.Server(415): Started @109191ms
2024-11-23T22:53:03,868 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:05,118 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data1/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:05,118 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data2/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:05,138 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:05,141 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6c6d791320eb23e with lease ID 0xfd8eb800fe13cefd: Processing first storage report for DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66 from datanode DatanodeRegistration(127.0.0.1:38865, datanodeUuid=08de4517-03d8-443a-8946-5b1dd17a4844, infoPort=33867, infoSecurePort=0, ipcPort=38719, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:05,141 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6c6d791320eb23e with lease ID 0xfd8eb800fe13cefd: from storage DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66 node DatanodeRegistration(127.0.0.1:38865, datanodeUuid=08de4517-03d8-443a-8946-5b1dd17a4844, infoPort=33867, infoSecurePort=0, ipcPort=38719, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:05,141 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6c6d791320eb23e with lease ID 0xfd8eb800fe13cefd: Processing first storage report for DS-f76842ac-24a0-43bd-a44f-cf21303689fa from datanode DatanodeRegistration(127.0.0.1:38865, datanodeUuid=08de4517-03d8-443a-8946-5b1dd17a4844, infoPort=33867, infoSecurePort=0, ipcPort=38719, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:05,141 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6c6d791320eb23e with lease ID 0xfd8eb800fe13cefd: from storage DS-f76842ac-24a0-43bd-a44f-cf21303689fa node DatanodeRegistration(127.0.0.1:38865, datanodeUuid=08de4517-03d8-443a-8946-5b1dd17a4844, infoPort=33867, infoSecurePort=0, ipcPort=38719, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:05,256 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data3/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:05,256 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data4/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:05,272 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:05,274 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x405c81e7fe8ad26b with lease ID 0xfd8eb800fe13cefe: Processing first storage report for DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591 from datanode DatanodeRegistration(127.0.0.1:46025, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=36381, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:05,274 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x405c81e7fe8ad26b with lease ID 0xfd8eb800fe13cefe: from storage DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591 node DatanodeRegistration(127.0.0.1:46025, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=36381, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:05,274 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x405c81e7fe8ad26b with lease ID 0xfd8eb800fe13cefe: Processing first storage report for DS-85226e8b-b716-4ae7-a760-8831ed584b44 from datanode DatanodeRegistration(127.0.0.1:46025, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=36381, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:05,274 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x405c81e7fe8ad26b with lease ID 0xfd8eb800fe13cefe: from storage DS-85226e8b-b716-4ae7-a760-8831ed584b44 node DatanodeRegistration(127.0.0.1:46025, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=36381, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:05,316 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110
2024-11-23T22:53:05,319 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/zookeeper_0, clientPort=62169, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:53:05,320 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62169
2024-11-23T22:53:05,320 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,322 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:05,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:05,334 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c with version=8
2024-11-23T22:53:05,334 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:53:05,337 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:05,338 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45639
2024-11-23T22:53:05,339 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45639 connecting to ZooKeeper ensemble=127.0.0.1:62169
2024-11-23T22:53:05,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456390x0, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:05,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45639-0x10169e68d520000 connected
2024-11-23T22:53:05,726 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,729 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,732 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:05,733 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c, hbase.cluster.distributed=false
2024-11-23T22:53:05,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:05,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45639
2024-11-23T22:53:05,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45639
2024-11-23T22:53:05,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45639
2024-11-23T22:53:05,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45639
2024-11-23T22:53:05,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45639
2024-11-23T22:53:05,760 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:53:05,761 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:05,762 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43859
2024-11-23T22:53:05,763 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43859 connecting to ZooKeeper ensemble=127.0.0.1:62169
2024-11-23T22:53:05,763 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,765 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438590x0, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:05,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438590x0, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:05,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43859-0x10169e68d520001 connected
2024-11-23T22:53:05,779 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:53:05,779 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:53:05,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:53:05,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:05,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43859
2024-11-23T22:53:05,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43859
2024-11-23T22:53:05,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43859
2024-11-23T22:53:05,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43859
2024-11-23T22:53:05,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43859
2024-11-23T22:53:05,799 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:45639
2024-11-23T22:53:05,799 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:05,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:05,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:05,810 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:05,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:53:05,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:05,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:05,821 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:53:05,821 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,45639,1732402385336 from backup master directory
2024-11-23T22:53:05,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:05,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:05,831 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:05,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:05,831 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:05,835 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/hbase.id] with ID: 85f9904b-8290-4a9b-8890-adf4debbcd46
2024-11-23T22:53:05,835 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/.tmp/hbase.id
2024-11-23T22:53:05,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:05,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:05,842 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/.tmp/hbase.id]:[hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/hbase.id]
2024-11-23T22:53:05,858 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:05,858 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:53:05,860 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-23T22:53:05,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:05,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:05,879 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:05,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:05,880 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:53:05,881 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:53:05,881 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:05,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:05,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:05,892 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store
2024-11-23T22:53:05,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:05,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:53:05,901 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:05,901 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:05,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402385901Disabling compacts and flushes for region at 1732402385901Disabling writes for close at 1732402385901Writing region close event to WAL at 1732402385901Closed at 1732402385901
2024-11-23T22:53:05,903 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/.initializing
2024-11-23T22:53:05,903 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:05,908 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C45639%2C1732402385336, suffix=, logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336, archiveDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/oldWALs, maxLogs=10
2024-11-23T22:53:05,908 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C45639%2C1732402385336.1732402385908
2024-11-23T22:53:05,915 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908
2024-11-23T22:53:05,917 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33867:33867),(127.0.0.1/127.0.0.1:36381:36381)]
2024-11-23T22:53:05,918 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:05,918 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:05,918 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,919 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,921 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:05,922 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:53:05,922 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:05,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:05,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:05,924 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:53:05,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:05,925 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:05,925 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:05,926 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:53:05,926 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:05,927 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:05,927 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:05,928 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:53:05,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:05,929 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:05,929 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,930 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,930 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,932 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,932 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,933 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:53:05,934 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:05,936 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:05,937 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836575, jitterRate=0.06376032531261444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:53:05,938 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402385919Initializing all the Stores at 1732402385920 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402385920Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402385921 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402385921Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402385921Cleaning up temporary data from old regions at 1732402385932 (+11 ms)Region opened successfully at 1732402385938 (+6 ms)
2024-11-23T22:53:05,938 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:53:05,942 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b17f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:05,942 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:53:05,943 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:53:05,943 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:53:05,943 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:53:05,943 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:53:05,944 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:53:05,944 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:53:05,946 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:53:05,947 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:53:05,975 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:53:05,975 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:53:05,977 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:53:05,989 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:53:05,989 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:53:05,992 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:53:05,999 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:53:06,001 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:53:06,009 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:53:06,012 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:53:06,020 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:53:06,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:06,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:06,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,031 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,45639,1732402385336, sessionid=0x10169e68d520000, setting cluster-up flag (Was=false)
2024-11-23T22:53:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,083 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:53:06,085 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:06,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,136 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:53:06,140 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:06,143 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:53:06,147 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:06,148 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:53:06,148 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:53:06,148 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,45639,1732402385336
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:53:06,150 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:06,150 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:06,150 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:06,150 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:06,151 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:53:06,151 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,151 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:06,151 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402416152
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:53:06,152 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,153 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:53:06,153 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:06,153 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:53:06,153 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:53:06,153 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:53:06,153 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:53:06,153 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:53:06,154 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402386154,5,FailOnTimeoutGroup]
2024-11-23T22:53:06,154 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402386154,5,FailOnTimeoutGroup]
2024-11-23T22:53:06,154 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,154 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,154 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:53:06,154 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,154 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,154 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:53:06,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:06,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:06,162 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:53:06,162 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c
2024-11-23T22:53:06,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:06,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:06,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:06,171 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:06,173 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:06,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,173 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,173 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:06,175 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:06,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,175 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:06,177 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:06,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:06,180 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:06,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:06,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740
2024-11-23T22:53:06,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740
2024-11-23T22:53:06,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:06,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:06,184 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:06,185 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(746): ClusterId : 85f9904b-8290-4a9b-8890-adf4debbcd46
2024-11-23T22:53:06,185 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:53:06,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:06,188 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:06,188 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707252, jitterRate=-0.10068260133266449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402386170Initializing all the Stores at 1732402386171 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386171Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386171Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402386171Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386171Cleaning up temporary data from old regions at 1732402386183 (+12 ms)Region opened successfully at 1732402386189 (+6 ms)
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:53:06,190 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:53:06,190 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:06,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402386190Disabling compacts and flushes for region at 1732402386190Disabling writes for close at 1732402386190Writing region close event to WAL at 1732402386190Closed at 1732402386190
2024-11-23T22:53:06,192 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:06,192 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:53:06,192 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:53:06,194 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:06,195 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:53:06,197 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:53:06,197 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:53:06,210 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:53:06,211 DEBUG [RS:0;dd96e8dc8369:43859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9c1512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:06,223 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:43859
2024-11-23T22:53:06,223 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:53:06,223 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:53:06,223 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:53:06,224 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,45639,1732402385336 with port=43859, startcode=1732402385760
2024-11-23T22:53:06,224 DEBUG [RS:0;dd96e8dc8369:43859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:53:06,226 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36107, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:53:06,227 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45639 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,227 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45639 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,229 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c
2024-11-23T22:53:06,229 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39715
2024-11-23T22:53:06,229 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:53:06,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:06,242 DEBUG [RS:0;dd96e8dc8369:43859 {}] zookeeper.ZKUtil(111): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,242 WARN  [RS:0;dd96e8dc8369:43859 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:06,242 INFO  [RS:0;dd96e8dc8369:43859 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:06,242 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,43859,1732402385760]
2024-11-23T22:53:06,242 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,246 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:53:06,248 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:53:06,248 INFO  [RS:0;dd96e8dc8369:43859 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:53:06,248 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,249 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:53:06,250 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:53:06,250 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,250 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,251 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:06,251 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:06,251 DEBUG [RS:0;dd96e8dc8369:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:06,251 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,277 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,277 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,277 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,277 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,277 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43859,1732402385760-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:06,301 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:53:06,301 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43859,1732402385760-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,301 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,301 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.Replication(171): dd96e8dc8369,43859,1732402385760 started
2024-11-23T22:53:06,326 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,326 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,43859,1732402385760, RpcServer on dd96e8dc8369/172.17.0.2:43859, sessionid=0x10169e68d520001
2024-11-23T22:53:06,327 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:53:06,327 DEBUG [RS:0;dd96e8dc8369:43859 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,327 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43859,1732402385760'
2024-11-23T22:53:06,327 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:53:06,327 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43859,1732402385760'
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:53:06,328 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:53:06,329 DEBUG [RS:0;dd96e8dc8369:43859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:53:06,329 INFO  [RS:0;dd96e8dc8369:43859 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:53:06,329 INFO  [RS:0;dd96e8dc8369:43859 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:53:06,346 WARN  [dd96e8dc8369:45639 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:53:06,431 INFO  [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C43859%2C1732402385760, suffix=, logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760, archiveDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs, maxLogs=32
2024-11-23T22:53:06,432 INFO  [RS:0;dd96e8dc8369:43859 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402386431
2024-11-23T22:53:06,438 INFO  [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
2024-11-23T22:53:06,439 DEBUG [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36381:36381),(127.0.0.1/127.0.0.1:33867:33867)]
2024-11-23T22:53:06,596 DEBUG [dd96e8dc8369:45639 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:53:06,597 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,600 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,43859,1732402385760, state=OPENING
2024-11-23T22:53:06,610 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:53:06,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:06,621 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:06,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:06,621 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43859,1732402385760}]
2024-11-23T22:53:06,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:06,776 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:53:06,781 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58883, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:53:06,788 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:53:06,788 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:06,790 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C43859%2C1732402385760.meta, suffix=.meta, logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760, archiveDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs, maxLogs=32
2024-11-23T22:53:06,791 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
2024-11-23T22:53:06,797 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
2024-11-23T22:53:06,798 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36381:36381),(127.0.0.1/127.0.0.1:33867:33867)]
2024-11-23T22:53:06,799 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:06,799 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:53:06,800 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:53:06,800 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:53:06,800 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:53:06,800 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:06,800 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:53:06,800 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:53:06,802 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:06,803 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:06,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,804 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,804 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:06,806 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:06,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,806 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,806 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:06,807 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:06,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:06,810 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:06,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:06,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:06,810 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:06,811 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740
2024-11-23T22:53:06,813 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740
2024-11-23T22:53:06,815 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:06,815 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:06,816 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:06,817 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:06,818 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715651, jitterRate=-0.09000368416309357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:06,818 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:53:06,819 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402386800Writing region info on filesystem at 1732402386801 (+1 ms)Initializing all the Stores at 1732402386802 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386802Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386802Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402386802Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402386802Cleaning up temporary data from old regions at 1732402386815 (+13 ms)Running coprocessor post-open hooks at 1732402386818 (+3 ms)Region opened successfully at 1732402386818
2024-11-23T22:53:06,820 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402386775
2024-11-23T22:53:06,822 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:53:06,822 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:53:06,823 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,824 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,43859,1732402385760, state=OPEN
2024-11-23T22:53:06,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:53:06,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:53:06,860 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:06,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:06,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:06,865 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:53:06,865 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,43859,1732402385760 in 240 msec
2024-11-23T22:53:06,868 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:53:06,868 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 673 msec
2024-11-23T22:53:06,869 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:06,869 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:53:06,871 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:53:06,871 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,43859,1732402385760, seqNum=-1]
2024-11-23T22:53:06,871 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:53:06,873 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54733, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:53:06,880 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 733 msec
2024-11-23T22:53:06,880 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402386880, completionTime=-1
2024-11-23T22:53:06,880 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:53:06,880 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:53:06,882 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:53:06,882 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402446882
2024-11-23T22:53:06,882 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402506882
2024-11-23T22:53:06,882 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:45639, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,883 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,885 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.056sec
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:06,887 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:53:06,889 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:53:06,889 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:53:06,890 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45639,1732402385336-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:06,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f683c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:06,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,45639,-1 for getting cluster id
2024-11-23T22:53:06,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:53:06,987 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '85f9904b-8290-4a9b-8890-adf4debbcd46'
2024-11-23T22:53:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:53:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "85f9904b-8290-4a9b-8890-adf4debbcd46"

2024-11-23T22:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a72ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,45639,-1]
2024-11-23T22:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:06,990 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:53:06,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261a63ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:53:06,991 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:53:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,43859,1732402385760, seqNum=-1]
2024-11-23T22:53:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:53:06,994 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:53:06,996 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:06,996 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:06,999 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:53:07,015 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:07,015 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:07,016 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36341
2024-11-23T22:53:07,018 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36341 connecting to ZooKeeper ensemble=127.0.0.1:62169
2024-11-23T22:53:07,018 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:07,019 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:07,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363410x0, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:07,042 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36341-0x10169e68d520002 connected
2024-11-23T22:53:07,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-23T22:53:07,042 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-23T22:53:07,043 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:53:07,043 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:53:07,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:53:07,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:07,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36341
2024-11-23T22:53:07,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36341
2024-11-23T22:53:07,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36341
2024-11-23T22:53:07,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36341
2024-11-23T22:53:07,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36341
2024-11-23T22:53:07,050 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(746): ClusterId : 85f9904b-8290-4a9b-8890-adf4debbcd46
2024-11-23T22:53:07,050 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:53:07,063 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:53:07,063 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:53:07,074 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:53:07,074 DEBUG [RS:1;dd96e8dc8369:36341 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c56f5f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:07,088 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd96e8dc8369:36341
2024-11-23T22:53:07,089 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:53:07,089 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:53:07,089 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:53:07,089 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,45639,1732402385336 with port=36341, startcode=1732402387015
2024-11-23T22:53:07,090 DEBUG [RS:1;dd96e8dc8369:36341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:53:07,091 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35649, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:53:07,091 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45639 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,091 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45639 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,093 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c
2024-11-23T22:53:07,093 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39715
2024-11-23T22:53:07,093 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:53:07,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:07,105 DEBUG [RS:1;dd96e8dc8369:36341 {}] zookeeper.ZKUtil(111): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,105 WARN  [RS:1;dd96e8dc8369:36341 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:07,105 INFO  [RS:1;dd96e8dc8369:36341 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:07,105 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,36341,1732402387015]
2024-11-23T22:53:07,105 DEBUG [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,110 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:53:07,112 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:53:07,112 INFO  [RS:1;dd96e8dc8369:36341 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:53:07,112 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,113 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:53:07,114 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:53:07,114 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,114 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,114 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,114 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,114 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:07,115 DEBUG [RS:1;dd96e8dc8369:36341 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,116 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36341,1732402387015-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:07,130 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:53:07,130 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36341,1732402387015-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,130 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,130 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.Replication(171): dd96e8dc8369,36341,1732402387015 started
2024-11-23T22:53:07,143 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:07,143 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,36341,1732402387015, RpcServer on dd96e8dc8369/172.17.0.2:36341, sessionid=0x10169e68d520002
2024-11-23T22:53:07,143 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:53:07,143 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;dd96e8dc8369:36341,5,FailOnTimeoutGroup]
2024-11-23T22:53:07,143 DEBUG [RS:1;dd96e8dc8369:36341 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,143 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,36341,1732402387015'
2024-11-23T22:53:07,143 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:53:07,144 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-23T22:53:07,144 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,36341,1732402387015'
2024-11-23T22:53:07,144 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:53:07,145 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:53:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:07,145 DEBUG [RS:1;dd96e8dc8369:36341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:53:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f09b82f
2024-11-23T22:53:07,145 INFO  [RS:1;dd96e8dc8369:36341 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:53:07,145 INFO  [RS:1;dd96e8dc8369:36341 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:53:07,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-23T22:53:07,147 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-23T22:53:07,148 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-23T22:53:07,148 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-23T22:53:07,148 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:53:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-23T22:53:07,151 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-23T22:53:07,151 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:07,151 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-23T22:53:07,152 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-23T22:53:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:53:07,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741835_1011 (size=393)
2024-11-23T22:53:07,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741835_1011 (size=393)
2024-11-23T22:53:07,161 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 377d9f4bf35a8f2c0bbf065ae6c0d147, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c
2024-11-23T22:53:07,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46025 is added to blk_1073741836_1012 (size=76)
2024-11-23T22:53:07,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38865 is added to blk_1073741836_1012 (size=76)
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 377d9f4bf35a8f2c0bbf065ae6c0d147, disabling compactions & flushes
2024-11-23T22:53:07,169 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147. after waiting 0 ms
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,169 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,169 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:
Waiting for close lock at 1732402387169Disabling compacts and flushes for region at 1732402387169Disabling writes for close at 1732402387169Writing region close event to WAL at 1732402387169Closed at 1732402387169
2024-11-23T22:53:07,171 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-23T22:53:07,171 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732402387171"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402387171"}]},"ts":"1732402387171"}
2024-11-23T22:53:07,174 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-23T22:53:07,176 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-23T22:53:07,176 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402387176"}]},"ts":"1732402387176"}
2024-11-23T22:53:07,178 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-23T22:53:07,179 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=377d9f4bf35a8f2c0bbf065ae6c0d147, ASSIGN}]
2024-11-23T22:53:07,180 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=377d9f4bf35a8f2c0bbf065ae6c0d147, ASSIGN
2024-11-23T22:53:07,181 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=377d9f4bf35a8f2c0bbf065ae6c0d147, ASSIGN; state=OFFLINE, location=dd96e8dc8369,43859,1732402385760; forceNewPlan=false, retain=false
2024-11-23T22:53:07,250 INFO  [RS:1;dd96e8dc8369:36341 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C36341%2C1732402387015, suffix=, logDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015, archiveDir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs, maxLogs=32
2024-11-23T22:53:07,251 INFO  [RS:1;dd96e8dc8369:36341 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C36341%2C1732402387015.1732402387251
2024-11-23T22:53:07,261 INFO  [RS:1;dd96e8dc8369:36341 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
2024-11-23T22:53:07,262 DEBUG [RS:1;dd96e8dc8369:36341 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36381:36381),(127.0.0.1/127.0.0.1:33867:33867)]
2024-11-23T22:53:07,332 INFO  [dd96e8dc8369:45639 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-23T22:53:07,333 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=377d9f4bf35a8f2c0bbf065ae6c0d147, regionState=OPENING, regionLocation=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:07,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=377d9f4bf35a8f2c0bbf065ae6c0d147, ASSIGN because future has completed
2024-11-23T22:53:07,339 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 377d9f4bf35a8f2c0bbf065ae6c0d147, server=dd96e8dc8369,43859,1732402385760}]
2024-11-23T22:53:07,504 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,504 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 377d9f4bf35a8f2c0bbf065ae6c0d147, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:07,506 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,506 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:07,506 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,506 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,509 INFO  [StoreOpener-377d9f4bf35a8f2c0bbf065ae6c0d147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 377d9f4bf35a8f2c0bbf065ae6c0d147 
2024-11-23T22:53:07,510 INFO  [StoreOpener-377d9f4bf35a8f2c0bbf065ae6c0d147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377d9f4bf35a8f2c0bbf065ae6c0d147 columnFamilyName info
2024-11-23T22:53:07,510 DEBUG [StoreOpener-377d9f4bf35a8f2c0bbf065ae6c0d147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:07,511 INFO  [StoreOpener-377d9f4bf35a8f2c0bbf065ae6c0d147-1 {}] regionserver.HStore(327): Store=377d9f4bf35a8f2c0bbf065ae6c0d147/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:07,511 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,512 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,513 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,513 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,513 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,515 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,518 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:07,519 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 377d9f4bf35a8f2c0bbf065ae6c0d147; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805070, jitterRate=0.0236998051404953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:53:07,519 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:07,520 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:
Running coprocessor pre-open hook at 1732402387506Writing region info on filesystem at 1732402387506Initializing all the Stores at 1732402387508 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402387508Cleaning up temporary data from old regions at 1732402387513 (+5 ms)Running coprocessor post-open hooks at 1732402387519 (+6 ms)Region opened successfully at 1732402387520 (+1 ms)
2024-11-23T22:53:07,521 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., pid=6, masterSystemTime=1732402387494
2024-11-23T22:53:07,525 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,525 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:07,527 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=377d9f4bf35a8f2c0bbf065ae6c0d147, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:07,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 377d9f4bf35a8f2c0bbf065ae6c0d147, server=dd96e8dc8369,43859,1732402385760 because future has completed
2024-11-23T22:53:07,536 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-23T22:53:07,536 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 377d9f4bf35a8f2c0bbf065ae6c0d147, server=dd96e8dc8369,43859,1732402385760 in 193 msec
2024-11-23T22:53:07,539 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-23T22:53:07,539 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=377d9f4bf35a8f2c0bbf065ae6c0d147, ASSIGN in 358 msec
2024-11-23T22:53:07,540 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-23T22:53:07,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402387540"}]},"ts":"1732402387540"}
2024-11-23T22:53:07,542 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-23T22:53:07,544 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-23T22:53:07,546 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 396 msec
2024-11-23T22:53:12,308 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:53:12,313 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:12,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:12,336 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:12,336 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:12,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:53:12,343 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-23T22:53:12,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-23T22:53:12,344 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-23T22:53:12,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:53:12,345 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-23T22:53:12,345 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-23T22:53:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45639 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:53:17,226 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-23T22:53:17,226 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-23T22:53:17,231 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-23T22:53:17,231 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:17,249 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:17,253 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:17,254 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:17,254 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:17,254 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:17,254 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a9ffe70{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:17,255 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45b83ed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:17,352 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d34c067{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-40157-hadoop-hdfs-3_4_1-tests_jar-_-any-14351780338832786110/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:17,353 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@40d66d7a{HTTP/1.1, (http/1.1)}{localhost:40157}
2024-11-23T22:53:17,353 INFO  [Time-limited test {}] server.Server(415): Started @122677ms
2024-11-23T22:53:17,354 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:17,397 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:17,401 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:17,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:17,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:17,405 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:53:17,408 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7076200d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:17,409 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d39bcd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:17,521 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7add9015{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-40533-hadoop-hdfs-3_4_1-tests_jar-_-any-919779625825578844/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:17,521 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@77d9f90{HTTP/1.1, (http/1.1)}{localhost:40533}
2024-11-23T22:53:17,522 INFO  [Time-limited test {}] server.Server(415): Started @122846ms
2024-11-23T22:53:17,523 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:17,562 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:17,566 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:17,567 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:17,567 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:17,567 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:53:17,568 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dfb8150{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:17,568 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a60480b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:17,711 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20e9d233{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-34135-hadoop-hdfs-3_4_1-tests_jar-_-any-6435137425452234273/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:17,711 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7072e3f6{HTTP/1.1, (http/1.1)}{localhost:34135}
2024-11-23T22:53:17,711 INFO  [Time-limited test {}] server.Server(415): Started @123036ms
2024-11-23T22:53:17,713 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:18,597 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data6/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,597 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data5/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,631 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:18,634 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27c1fa30c15860c1 with lease ID 0xfd8eb800fe13ceff: Processing first storage report for DS-ac7b0262-4c40-4940-9081-592113fa766e from datanode DatanodeRegistration(127.0.0.1:35033, datanodeUuid=68bba244-852d-4e51-9b89-3757625f5e23, infoPort=33833, infoSecurePort=0, ipcPort=42011, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,634 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27c1fa30c15860c1 with lease ID 0xfd8eb800fe13ceff: from storage DS-ac7b0262-4c40-4940-9081-592113fa766e node DatanodeRegistration(127.0.0.1:35033, datanodeUuid=68bba244-852d-4e51-9b89-3757625f5e23, infoPort=33833, infoSecurePort=0, ipcPort=42011, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:18,634 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27c1fa30c15860c1 with lease ID 0xfd8eb800fe13ceff: Processing first storage report for DS-db498d92-e1e6-489e-bbe6-927e17321844 from datanode DatanodeRegistration(127.0.0.1:35033, datanodeUuid=68bba244-852d-4e51-9b89-3757625f5e23, infoPort=33833, infoSecurePort=0, ipcPort=42011, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,634 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27c1fa30c15860c1 with lease ID 0xfd8eb800fe13ceff: from storage DS-db498d92-e1e6-489e-bbe6-927e17321844 node DatanodeRegistration(127.0.0.1:35033, datanodeUuid=68bba244-852d-4e51-9b89-3757625f5e23, infoPort=33833, infoSecurePort=0, ipcPort=42011, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-23T22:53:18,809 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data7/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,809 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data8/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,833 WARN  [Thread-831 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:18,835 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a57c4aaeeb82b6d with lease ID 0xfd8eb800fe13cf00: Processing first storage report for DS-ca74df64-83b6-450f-bf7f-c2584557fa2b from datanode DatanodeRegistration(127.0.0.1:40591, datanodeUuid=bbd12c06-39a7-4757-a0a2-d080a179cf0d, infoPort=35767, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,835 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a57c4aaeeb82b6d with lease ID 0xfd8eb800fe13cf00: from storage DS-ca74df64-83b6-450f-bf7f-c2584557fa2b node DatanodeRegistration(127.0.0.1:40591, datanodeUuid=bbd12c06-39a7-4757-a0a2-d080a179cf0d, infoPort=35767, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:18,836 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a57c4aaeeb82b6d with lease ID 0xfd8eb800fe13cf00: Processing first storage report for DS-1f7f98b8-f044-4752-998f-0b0d19840cb7 from datanode DatanodeRegistration(127.0.0.1:40591, datanodeUuid=bbd12c06-39a7-4757-a0a2-d080a179cf0d, infoPort=35767, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,836 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a57c4aaeeb82b6d with lease ID 0xfd8eb800fe13cf00: from storage DS-1f7f98b8-f044-4752-998f-0b0d19840cb7 node DatanodeRegistration(127.0.0.1:40591, datanodeUuid=bbd12c06-39a7-4757-a0a2-d080a179cf0d, infoPort=35767, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:18,953 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,953 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10/current/BP-942195297-172.17.0.2-1732402382938/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:18,979 WARN  [Thread-853 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:18,982 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4648e94a82006f7b with lease ID 0xfd8eb800fe13cf01: Processing first storage report for DS-69e4986e-7e81-4236-a28c-7fdb8717ab30 from datanode DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,982 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4648e94a82006f7b with lease ID 0xfd8eb800fe13cf01: from storage DS-69e4986e-7e81-4236-a28c-7fdb8717ab30 node DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-23T22:53:18,982 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4648e94a82006f7b with lease ID 0xfd8eb800fe13cf01: Processing first storage report for DS-9100961e-fccd-409a-a2c2-7371d43346c3 from datanode DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938)
2024-11-23T22:53:18,982 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4648e94a82006f7b with lease ID 0xfd8eb800fe13cf01: from storage DS-9100961e-fccd-409a-a2c2-7371d43346c3 node DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:19,057 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,057 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,057 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,058 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,058 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 block BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:19,058 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:19,058 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 block BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:19,058 WARN  [PacketResponder: BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46025] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,058 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:19,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1081356134_22 at /127.0.0.1:38366 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38366 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:58010 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46025:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58010 dst: /127.0.0.1:46025
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_788849474_22 at /127.0.0.1:38434 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38434 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:38386 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38386 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a02718e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:19,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_788849474_22 at /127.0.0.1:58034 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:46025:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58034 dst: /127.0.0.1:46025
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1081356134_22 at /127.0.0.1:57976 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46025:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57976 dst: /127.0.0.1:46025
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:58026 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46025:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58026 dst: /127.0.0.1:46025
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:38394 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38394 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,061 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@527a79d3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:19,061 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:19,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acad2a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:19,062 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36318c9a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:19,063 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:19,063 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:19,063 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:19,063 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid 969d3a7d-7768-473e-9942-01ee0825f8da) service to localhost/127.0.0.1:39715
2024-11-23T22:53:19,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data3/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:19,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data4/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:19,064 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:19,066 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@6b6d8dd5 {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing unknown operation  src: /127.0.0.1:36542 dst: /127.0.0.1:38865
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,066 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 block BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36540 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36540 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,069 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36534 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36534 dst: /127.0.0.1:38865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:19,070 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 block BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,073 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,073 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741834_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,074 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f0e057c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:19,075 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26c6facf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:19,075 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:19,075 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34786aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:19,075 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@596eb72c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:19,076 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:19,076 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:19,076 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid 08de4517-03d8-443a-8946-5b1dd17a4844) service to localhost/127.0.0.1:39715
2024-11-23T22:53:19,076 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:19,077 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data1/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:19,077 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data2/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:19,077 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., hostname=dd96e8dc8369,43859,1732402385760, seqNum=2]
2024-11-23T22:53:19,083 ERROR [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c-prefix:dd96e8dc8369,43859,1732402385760 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,083 WARN  [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c-prefix:dd96e8dc8369,43859,1732402385760 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,084 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C43859%2C1732402385760:(num 1732402386431) roll requested
2024-11-23T22:53:19,084 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402399084
2024-11-23T22:53:19,087 WARN  [Thread-908 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,087 WARN  [Thread-908 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:19,087 WARN  [Thread-908 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741838_1018
2024-11-23T22:53:19,090 WARN  [Thread-908 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:19,096 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:19,097 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:19,097 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:19,097 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:19,097 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:19,097 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084
2024-11-23T22:53:19,097 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,098 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,098 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-23T22:53:19,099 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-23T22:53:19,099 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
2024-11-23T22:53:19,100 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35767:35767),(127.0.0.1/127.0.0.1:33833:33833)]
2024-11-23T22:53:19,100 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:19,102 WARN  [IPC Server handler 3 on default port 39715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1015
2024-11-23T22:53:19,105 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 after 4ms
2024-11-23T22:53:19,117 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:19,883 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:21,100 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:21,102 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084
2024-11-23T22:53:21,102 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:21,103 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084 block BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:21,103 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:52652 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:40591:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52652 dst: /127.0.0.1:40591
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:21,103 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:35366 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35366 dst: /127.0.0.1:35033
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:21,117 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:21,283 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7add9015{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:21,284 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@77d9f90{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:21,284 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:21,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d39bcd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:21,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7076200d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:21,286 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:21,286 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid bbd12c06-39a7-4757-a0a2-d080a179cf0d) service to localhost/127.0.0.1:39715
2024-11-23T22:53:21,286 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:21,286 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:21,314 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data8/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:21,315 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data7/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:21,315 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:21,883 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:23,101 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:23,101 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]
2024-11-23T22:53:23,101 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C43859%2C1732402385760:(num 1732402399084) roll requested
2024-11-23T22:53:23,102 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402403102
2024-11-23T22:53:23,105 WARN  [Thread-917 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:23,105 WARN  [Thread-917 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:23,105 WARN  [Thread-917 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741840_1022
2024-11-23T22:53:23,106 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 after 4007ms
2024-11-23T22:53:23,106 WARN  [Thread-917 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:23,112 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:23,112 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:23,112 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:23,112 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:23,112 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:23,112 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402403102
2024-11-23T22:53:23,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741839_1021 (size=2431)
2024-11-23T22:53:23,115 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33833:33833),(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:23,115 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:23,115 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084 is not closed yet, will try archiving it next time
2024-11-23T22:53:23,117 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:23,320 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-23T22:53:23,517 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:23,884 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:24,645 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3cfd0faa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35033, datanodeUuid=68bba244-852d-4e51-9b89-3757625f5e23, infoPort=33833, infoSecurePort=0, ipcPort=42011, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741839_1021 to 127.0.0.1:40591 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,116 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,118 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,324 WARN  [ResponseProcessor for block BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,325 WARN  [DataStreamer for file /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402403102 block BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:25,325 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:60870 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60870 dst: /127.0.0.1:35033
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,325 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36498 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36498 dst: /127.0.0.1:35683
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d34c067{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:25,369 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@40d66d7a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:25,369 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:25,369 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45b83ed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:25,369 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a9ffe70{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:25,371 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:25,371 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:25,371 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid 68bba244-852d-4e51-9b89-3757625f5e23) service to localhost/127.0.0.1:39715
2024-11-23T22:53:25,371 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:25,371 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data5/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:25,372 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data6/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:25,372 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:25,383 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:53:25,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/92faaa153ecd4e32ae5cf18561c99219 is 1080, key is row0002/info:/1732402401317/Put/seqid=0
2024-11-23T22:53:25,406 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,406 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:25,406 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741842_1025
2024-11-23T22:53:25,406 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:25,409 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36522 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026 to mirror 127.0.0.1:46025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,410 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36522 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:25,410 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36522 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36522 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,412 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46025
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,413 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:25,413 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741843_1026
2024-11-23T22:53:25,413 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:25,416 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38865
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,416 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36538 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027 to mirror 127.0.0.1:38865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,416 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:25,417 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027
2024-11-23T22:53:25,417 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36538 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:25,417 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36538 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36538 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,417 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:25,420 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:25,420 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36552 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028 to mirror 127.0.0.1:35033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,420 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:25,420 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028
2024-11-23T22:53:25,420 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36552 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:25,420 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36552 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36552 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:25,421 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:25,422 WARN  [IPC Server handler 3 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:25,422 WARN  [IPC Server handler 3 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:25,422 WARN  [IPC Server handler 3 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:25,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741846_1029 (size=10347)
2024-11-23T22:53:25,827 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/92faaa153ecd4e32ae5cf18561c99219
2024-11-23T22:53:25,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/92faaa153ecd4e32ae5cf18561c99219 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219
2024-11-23T22:53:25,845 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219, entries=5, sequenceid=11, filesize=10.1 K
2024-11-23T22:53:25,847 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 464ms, sequenceid=11, compaction requested=false
2024-11-23T22:53:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:25,884 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:26,009 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-23T22:53:26,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/aa789228bd2f483e8cbfacd978ad0626 is 1080, key is row0007/info:/1732402405384/Put/seqid=0
2024-11-23T22:53:26,018 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:26,018 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:26,018 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741847_1030
2024-11-23T22:53:26,018 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:26,019 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:26,019 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:26,019 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741848_1031
2024-11-23T22:53:26,020 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:26,021 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:26,021 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:26,021 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741849_1032
2024-11-23T22:53:26,022 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:26,023 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:26,023 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:26,023 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741850_1033
2024-11-23T22:53:26,023 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:26,024 WARN  [IPC Server handler 2 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:26,024 WARN  [IPC Server handler 2 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:26,024 WARN  [IPC Server handler 2 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:26,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741851_1034 (size=12506)
2024-11-23T22:53:26,432 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/aa789228bd2f483e8cbfacd978ad0626
2024-11-23T22:53:26,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/aa789228bd2f483e8cbfacd978ad0626 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626
2024-11-23T22:53:26,447 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626, entries=7, sequenceid=24, filesize=12.2 K
2024-11-23T22:53:26,449 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 440ms, sequenceid=24, compaction requested=false
2024-11-23T22:53:26,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:26,449 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-23T22:53:26,449 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:26,449 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626 because midkey is the same as first or last row
2024-11-23T22:53:27,116 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,116 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]
2024-11-23T22:53:27,116 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C43859%2C1732402385760:(num 1732402403102) roll requested
2024-11-23T22:53:27,117 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402407116
2024-11-23T22:53:27,118 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,120 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,120 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:27,120 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741852_1035
2024-11-23T22:53:27,121 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:27,124 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40591
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036 to mirror 127.0.0.1:40591
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,124 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:27,125 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036
2024-11-23T22:53:27,125 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-23T22:53:27,125 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36598 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,125 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:27,128 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,129 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:27,129 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741854_1037
2024-11-23T22:53:27,130 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:27,133 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38865
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36606 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038 to mirror 127.0.0.1:38865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,133 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:27,133 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038
2024-11-23T22:53:27,133 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36606 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-23T22:53:27,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36606 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36606 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,134 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:27,135 WARN  [IPC Server handler 4 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:27,135 WARN  [IPC Server handler 4 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:27,135 WARN  [IPC Server handler 4 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:27,140 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:27,140 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:27,140 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:27,140 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:27,141 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:27,141 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402403102 with entries=24, filesize=24.23 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402407116
2024-11-23T22:53:27,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741841_1024 (size=24824)
2024-11-23T22:53:27,145 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:27,145 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:27,145 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402403102 is not closed yet, will try archiving it next time
2024-11-23T22:53:27,146 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402399084 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs/dd96e8dc8369%2C43859%2C1732402385760.1732402399084
2024-11-23T22:53:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:27,432 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-23T22:53:27,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/2b81e68a8d2f4195ac9a8187002efb85 is 1079, key is tmprow/info:/1732402407431/Put/seqid=0
2024-11-23T22:53:27,444 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,445 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:27,445 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741857_1040
2024-11-23T22:53:27,446 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:27,449 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,449 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:27,449 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741858_1041
2024-11-23T22:53:27,450 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:27,452 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,452 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:27,452 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741859_1042
2024-11-23T22:53:27,453 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:27,455 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38865
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,455 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36634 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043 to mirror 127.0.0.1:38865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,456 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:27,456 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043
2024-11-23T22:53:27,456 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36634 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:27,456 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36634 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36634 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,456 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:27,458 WARN  [IPC Server handler 1 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:27,458 WARN  [IPC Server handler 1 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:27,458 WARN  [IPC Server handler 1 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:27,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741861_1044 (size=6027)
2024-11-23T22:53:27,543 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:27,863 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/2b81e68a8d2f4195ac9a8187002efb85
2024-11-23T22:53:27,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/2b81e68a8d2f4195ac9a8187002efb85 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85
2024-11-23T22:53:27,878 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85, entries=1, sequenceid=34, filesize=5.9 K
2024-11-23T22:53:27,880 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 447ms, sequenceid=34, compaction requested=true
2024-11-23T22:53:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-23T22:53:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626 because midkey is the same as first or last row
2024-11-23T22:53:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 377d9f4bf35a8f2c0bbf065ae6c0d147:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:53:27,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:27,881 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:53:27,882 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:53:27,883 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1541): 377d9f4bf35a8f2c0bbf065ae6c0d147/info is initiating minor compaction (all files)
2024-11-23T22:53:27,883 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 377d9f4bf35a8f2c0bbf065ae6c0d147/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:27,883 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85] into tmpdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp, totalSize=28.2 K
2024-11-23T22:53:27,883 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92faaa153ecd4e32ae5cf18561c99219, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732402401317
2024-11-23T22:53:27,884 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa789228bd2f483e8cbfacd978ad0626, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732402405384
2024-11-23T22:53:27,884 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b81e68a8d2f4195ac9a8187002efb85, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732402407431
2024-11-23T22:53:27,884 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,908 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 377d9f4bf35a8f2c0bbf065ae6c0d147#info#compaction#21 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:53:27,908 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/45412edee662459795feca520fa126ed is 1080, key is row0002/info:/1732402401317/Put/seqid=0
2024-11-23T22:53:27,920 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38865
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,920 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36658 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045 to mirror 127.0.0.1:38865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,920 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:27,920 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36658 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:27,920 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045
2024-11-23T22:53:27,920 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36658 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36658 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,921 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:27,922 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,922 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:27,922 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741863_1046
2024-11-23T22:53:27,923 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:27,925 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,925 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36662 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047 to mirror 127.0.0.1:35033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,925 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:27,926 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047
2024-11-23T22:53:27,926 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36662 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:27,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36662 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36662 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,926 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:27,929 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40591
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:27,929 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36666 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048 to mirror 127.0.0.1:40591
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,929 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:27,929 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36666 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:27,929 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048
2024-11-23T22:53:27,929 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36666 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36666 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,930 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:27,930 WARN  [IPC Server handler 2 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:27,930 WARN  [IPC Server handler 2 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:27,931 WARN  [IPC Server handler 2 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:27,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741866_1049 (size=17994)
2024-11-23T22:53:27,984 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7fd940b0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741846_1029 to 127.0.0.1:40591 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:27,984 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@711eb13b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741851_1034 to 127.0.0.1:38865 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:28,342 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/45412edee662459795feca520fa126ed as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed
2024-11-23T22:53:28,349 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 377d9f4bf35a8f2c0bbf065ae6c0d147/info of 377d9f4bf35a8f2c0bbf065ae6c0d147 into 45412edee662459795feca520fa126ed(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:53:28,349 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:28,350 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., storeName=377d9f4bf35a8f2c0bbf065ae6c0d147/info, priority=13, startTime=1732402407880; duration=0sec
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed because midkey is the same as first or last row
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed because midkey is the same as first or last row
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed because midkey is the same as first or last row
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:28,350 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 377d9f4bf35a8f2c0bbf065ae6c0d147:info
2024-11-23T22:53:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:28,855 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-23T22:53:28,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/4c57adc4846b459dbdb6fe75164fbdaa is 1079, key is tmprow/info:/1732402408854/Put/seqid=0
2024-11-23T22:53:28,861 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:28,861 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:28,862 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741867_1050
2024-11-23T22:53:28,862 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:28,864 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:28,864 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:28,864 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741868_1051
2024-11-23T22:53:28,865 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:28,866 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:28,866 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:28,866 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741869_1052
2024-11-23T22:53:28,867 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:28,869 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:28,869 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:28,869 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741870_1053
2024-11-23T22:53:28,869 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:28,870 WARN  [IPC Server handler 4 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:28,870 WARN  [IPC Server handler 4 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:28,871 WARN  [IPC Server handler 4 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:28,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741871_1054 (size=6027)
2024-11-23T22:53:28,985 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@711eb13b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741861_1044 to 127.0.0.1:35033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:28,985 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7fd940b0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741841_1024 to 127.0.0.1:38865 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:29,118 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,146 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,146 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]
2024-11-23T22:53:29,146 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C43859%2C1732402385760:(num 1732402407116) roll requested
2024-11-23T22:53:29,147 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402409146
2024-11-23T22:53:29,153 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,153 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:29,153 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741872_1055
2024-11-23T22:53:29,154 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:29,155 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,155 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:29,155 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741873_1056
2024-11-23T22:53:29,156 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:29,157 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,157 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:29,157 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741874_1057
2024-11-23T22:53:29,158 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:29,160 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46025
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:29,160 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36702 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058 to mirror 127.0.0.1:46025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:29,160 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:29,160 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058
2024-11-23T22:53:29,160 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36702 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-23T22:53:29,160 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36702 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36702 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:29,161 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:29,162 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:29,162 WARN  [IPC Server handler 0 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:29,162 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:29,170 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:29,170 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:29,170 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:29,171 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:29,171 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:29,172 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402407116 with entries=14, filesize=12.92 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402409146
2024-11-23T22:53:29,174 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:29,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741856_1039 (size=13234)
2024-11-23T22:53:29,174 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:29,174 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402407116 is not closed yet, will try archiving it next time
2024-11-23T22:53:29,174 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402403102 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs/dd96e8dc8369%2C43859%2C1732402385760.1732402403102
2024-11-23T22:53:29,189 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:29,275 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/4c57adc4846b459dbdb6fe75164fbdaa
2024-11-23T22:53:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/4c57adc4846b459dbdb6fe75164fbdaa as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa
2024-11-23T22:53:29,290 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa, entries=1, sequenceid=45, filesize=5.9 K
2024-11-23T22:53:29,292 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 436ms, sequenceid=45, compaction requested=false
2024-11-23T22:53:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:29,293 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-23T22:53:29,293 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:29,293 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed because midkey is the same as first or last row
2024-11-23T22:53:29,885 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:30,279 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-23T22:53:30,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dfbced2edeb2417b909ac9b3c602e92a is 1079, key is tmprow/info:/1732402410277/Put/seqid=0
2024-11-23T22:53:30,289 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40591
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,289 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36728 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060 to mirror 127.0.0.1:40591
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:30,289 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:30,289 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060
2024-11-23T22:53:30,289 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36728 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:30,290 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36728 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36728 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:30,290 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:30,292 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,292 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:30,292 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741878_1061
2024-11-23T22:53:30,294 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:30,296 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38865
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,296 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36740 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062 to mirror 127.0.0.1:38865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:30,296 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:30,296 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062
2024-11-23T22:53:30,297 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36740 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-23T22:53:30,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:36740 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36740 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:30,297 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:30,299 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,299 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:30,299 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741880_1063
2024-11-23T22:53:30,299 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:30,300 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:30,300 WARN  [IPC Server handler 0 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:30,300 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:30,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741881_1064 (size=6027)
2024-11-23T22:53:30,713 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dfbced2edeb2417b909ac9b3c602e92a
2024-11-23T22:53:30,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dfbced2edeb2417b909ac9b3c602e92a as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a
2024-11-23T22:53:30,725 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a, entries=1, sequenceid=55, filesize=5.9 K
2024-11-23T22:53:30,726 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 448ms, sequenceid=55, compaction requested=true
2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed because midkey is the same as first or last row
2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 377d9f4bf35a8f2c0bbf065ae6c0d147:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:53:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:30,726 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:53:30,728 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:53:30,728 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1541): 377d9f4bf35a8f2c0bbf065ae6c0d147/info is initiating minor compaction (all files)
2024-11-23T22:53:30,728 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 377d9f4bf35a8f2c0bbf065ae6c0d147/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:30,728 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a] into tmpdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp, totalSize=29.3 K
2024-11-23T22:53:30,729 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45412edee662459795feca520fa126ed, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732402401317
2024-11-23T22:53:30,729 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c57adc4846b459dbdb6fe75164fbdaa, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732402408854
2024-11-23T22:53:30,730 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbced2edeb2417b909ac9b3c602e92a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732402410277
2024-11-23T22:53:30,750 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 377d9f4bf35a8f2c0bbf065ae6c0d147#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:53:30,750 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/8df82d9f29c348969819fc35c3ddcfd4 is 1080, key is row0002/info:/1732402401317/Put/seqid=0
2024-11-23T22:53:30,752 WARN  [Thread-978 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,752 WARN  [Thread-978 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]) is bad.
2024-11-23T22:53:30,752 WARN  [Thread-978 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741882_1065
2024-11-23T22:53:30,753 WARN  [Thread-978 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46025,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]
2024-11-23T22:53:30,754 WARN  [Thread-978 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,754 WARN  [Thread-978 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:30,754 WARN  [Thread-978 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741883_1066
2024-11-23T22:53:30,755 WARN  [Thread-978 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:30,757 WARN  [Thread-978 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,757 WARN  [Thread-978 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]) is bad.
2024-11-23T22:53:30,757 WARN  [Thread-978 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741884_1067
2024-11-23T22:53:30,757 WARN  [Thread-978 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40591,DS-ca74df64-83b6-450f-bf7f-c2584557fa2b,DISK]
2024-11-23T22:53:30,759 WARN  [Thread-978 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:30,759 WARN  [Thread-978 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:30,759 WARN  [Thread-978 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741885_1068
2024-11-23T22:53:30,760 WARN  [Thread-978 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:30,760 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-23T22:53:30,760 WARN  [IPC Server handler 0 on default port 39715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-23T22:53:30,761 WARN  [IPC Server handler 0 on default port 39715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-23T22:53:30,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741886_1069 (size=18097)
2024-11-23T22:53:30,985 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7fd940b0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741866_1049 to 127.0.0.1:38865 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:30,985 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@711eb13b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741871_1054 to 127.0.0.1:35033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:31,119 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:31,172 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/8df82d9f29c348969819fc35c3ddcfd4 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4
2024-11-23T22:53:31,174 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:31,175 WARN  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-23T22:53:31,180 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 377d9f4bf35a8f2c0bbf065ae6c0d147/info of 377d9f4bf35a8f2c0bbf065ae6c0d147 into 8df82d9f29c348969819fc35c3ddcfd4(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:53:31,180 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:31,181 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., storeName=377d9f4bf35a8f2c0bbf065ae6c0d147/info, priority=13, startTime=1732402410726; duration=0sec
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 because midkey is the same as first or last row
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 because midkey is the same as first or last row
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 because midkey is the same as first or last row
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:31,181 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 377d9f4bf35a8f2c0bbf065ae6c0d147:info
2024-11-23T22:53:31,300 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:31,304 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:31,304 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:31,304 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:31,304 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:31,305 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44bf135d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:31,305 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ef1ec46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:31,407 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3af3a5c0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/java.io.tmpdir/jetty-localhost-33725-hadoop-hdfs-3_4_1-tests_jar-_-any-7657553414842131537/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:31,407 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@67e24eab{HTTP/1.1, (http/1.1)}{localhost:33725}
2024-11-23T22:53:31,407 INFO  [Time-limited test {}] server.Server(415): Started @136732ms
2024-11-23T22:53:31,409 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:31,830 WARN  [Thread-996 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:31,834 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a398947882dbca with lease ID 0xfd8eb800fe13cf02: from storage DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591 node DatanodeRegistration(127.0.0.1:45915, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=39203, infoSecurePort=0, ipcPort=42363, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 6, hasStaleStorage: false, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-23T22:53:31,835 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a398947882dbca with lease ID 0xfd8eb800fe13cf02: from storage DS-85226e8b-b716-4ae7-a760-8831ed584b44 node DatanodeRegistration(127.0.0.1:45915, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=39203, infoSecurePort=0, ipcPort=42363, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:31,885 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:31,984 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7fd940b0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741881_1064 to 127.0.0.1:40591 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:31,984 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@711eb13b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=f662443b-02eb-4dfb-847c-b712769e12a5, infoPort=39717, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741856_1039 to 127.0.0.1:38865 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:33,119 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:33,175 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:33,885 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:33,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741886_1069 (size=18097)
2024-11-23T22:53:35,119 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:35,175 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:35,315 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:53:35,886 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,153 ERROR [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData-prefix:dd96e8dc8369,45639,1732402385336 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,154 WARN  [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData-prefix:dd96e8dc8369,45639,1732402385336 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,154 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C45639%2C1732402385336:(num 1732402385908) roll requested
2024-11-23T22:53:36,154 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C45639%2C1732402385336.1732402416154
2024-11-23T22:53:36,157 WARN  [Thread-1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,158 WARN  [Thread-1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK], DatanodeInfoWithStorage[127.0.0.1:45915,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]) is bad.
2024-11-23T22:53:36,158 WARN  [Thread-1015 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741887_1070
2024-11-23T22:53:36,158 WARN  [Thread-1015 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]
2024-11-23T22:53:36,159 WARN  [Thread-1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,159 WARN  [Thread-1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:36,159 WARN  [Thread-1015 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741888_1071
2024-11-23T22:53:36,160 WARN  [Thread-1015 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:36,167 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:36,167 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:36,167 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:36,167 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:36,167 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:36,167 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 with entries=54, filesize=26.64 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402416154
2024-11-23T22:53:36,168 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,168 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:36,168 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908
2024-11-23T22:53:36,168 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39203:39203),(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:36,168 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 is not closed yet, will try archiving it next time
2024-11-23T22:53:36,168 WARN  [IPC Server handler 0 on default port 39715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-11-23T22:53:36,169 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 after 1ms
2024-11-23T22:53:37,120 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:37,176 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:39,120 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:39,176 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:40,170 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/WALs/dd96e8dc8369,45639,1732402385336/dd96e8dc8369%2C45639%2C1732402385336.1732402385908 after 4002ms
2024-11-23T22:53:41,120 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:41,176 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:43,121 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:43,177 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:44,640 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.1732402424640
2024-11-23T22:53:44,649 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:44,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1081356134_22 at /127.0.0.1:53816 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data4]'}, localName='127.0.0.1:45915', datanodeUuid='969d3a7d-7768-473e-9942-01ee0825f8da', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074 to mirror 127.0.0.1:35033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:44,649 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45915,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:44,649 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074
2024-11-23T22:53:44,649 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1081356134_22 at /127.0.0.1:53816 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-23T22:53:44,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1081356134_22 at /127.0.0.1:53816 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741890_1074] {}] datanode.DataXceiver(331): 127.0.0.1:45915:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53816 dst: /127.0.0.1:45915
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:44,650 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:44,654 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:44,654 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:44,655 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:44,655 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:44,655 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:44,655 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402409146 with entries=14, filesize=12.95 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402424640
2024-11-23T22:53:44,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39203:39203),(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:44,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:44,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402409146 is not closed yet, will try archiving it next time
2024-11-23T22:53:44,656 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402407116 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs/dd96e8dc8369%2C43859%2C1732402385760.1732402407116
2024-11-23T22:53:44,657 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741876_1059 (size=13268)
2024-11-23T22:53:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:44,664 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-23T22:53:44,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/8196ef38551d405a9e188d1003f3a146 is 1080, key is row0013/info:/1732402424658/Put/seqid=0
2024-11-23T22:53:44,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741892_1076 (size=9267)
2024-11-23T22:53:44,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741892_1076 (size=9267)
2024-11-23T22:53:44,679 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/8196ef38551d405a9e188d1003f3a146
2024-11-23T22:53:44,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/8196ef38551d405a9e188d1003f3a146 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146
2024-11-23T22:53:44,694 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146, entries=4, sequenceid=66, filesize=9.0 K
2024-11-23T22:53:44,695 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 32ms, sequenceid=66, compaction requested=false
2024-11-23T22:53:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.7 K, sizeToCheck=16.0 K
2024-11-23T22:53:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 because midkey is the same as first or last row
2024-11-23T22:53:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43859 {}] regionserver.HRegion(8855): Flush requested on 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:44,887 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-23T22:53:44,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dd90066588b04e96b66bdd7f7ddc7b21 is 1080, key is row0016/info:/1732402424665/Put/seqid=0
2024-11-23T22:53:44,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741893_1077 (size=13583)
2024-11-23T22:53:44,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741893_1077 (size=13583)
2024-11-23T22:53:44,908 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dd90066588b04e96b66bdd7f7ddc7b21
2024-11-23T22:53:44,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/dd90066588b04e96b66bdd7f7ddc7b21 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21
2024-11-23T22:53:44,926 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21, entries=8, sequenceid=78, filesize=13.3 K
2024-11-23T22:53:44,928 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9682, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 41ms, sequenceid=78, compaction requested=true
2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 because midkey is the same as first or last row
2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 377d9f4bf35a8f2c0bbf065ae6c0d147:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:53:44,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:44,928 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:53:44,930 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:53:44,930 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1541): 377d9f4bf35a8f2c0bbf065ae6c0d147/info is initiating minor compaction (all files)
2024-11-23T22:53:44,930 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 377d9f4bf35a8f2c0bbf065ae6c0d147/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:44,930 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21] into tmpdir=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp, totalSize=40.0 K
2024-11-23T22:53:44,931 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8df82d9f29c348969819fc35c3ddcfd4, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732402401317
2024-11-23T22:53:44,931 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8196ef38551d405a9e188d1003f3a146, keycount=4, bloomtype=ROW, size=9.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732402411089
2024-11-23T22:53:44,932 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd90066588b04e96b66bdd7f7ddc7b21, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732402424665
2024-11-23T22:53:44,948 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 377d9f4bf35a8f2c0bbf065ae6c0d147#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:53:44,948 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/83aa380290b945009f307f15c878f3a9 is 1080, key is row0002/info:/1732402401317/Put/seqid=0
2024-11-23T22:53:44,950 WARN  [Thread-1046 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:44,950 WARN  [Thread-1046 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:45915,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:44,950 WARN  [Thread-1046 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741894_1078
2024-11-23T22:53:44,951 WARN  [Thread-1046 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:44,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741895_1079 (size=28989)
2024-11-23T22:53:44,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741895_1079 (size=28989)
2024-11-23T22:53:44,963 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/83aa380290b945009f307f15c878f3a9 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/83aa380290b945009f307f15c878f3a9
2024-11-23T22:53:44,970 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 377d9f4bf35a8f2c0bbf065ae6c0d147/info of 377d9f4bf35a8f2c0bbf065ae6c0d147 into 83aa380290b945009f307f15c878f3a9(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:53:44,970 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:

2024-11-23T22:53:44,970 INFO  [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., storeName=377d9f4bf35a8f2c0bbf065ae6c0d147/info, priority=13, startTime=1732402424928; duration=0sec
2024-11-23T22:53:44,970 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-23T22:53:44,970 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/83aa380290b945009f307f15c878f3a9 because midkey is the same as first or last row
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/83aa380290b945009f307f15c878f3a9 because midkey is the same as first or last row
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/83aa380290b945009f307f15c878f3a9 because midkey is the same as first or last row
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:53:44,971 DEBUG [RS:0;dd96e8dc8369:43859-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 377d9f4bf35a8f2c0bbf065ae6c0d147:info
2024-11-23T22:53:45,058 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 is not closed yet, will try archiving it next time
2024-11-23T22:53:45,058 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402409146 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs/dd96e8dc8369%2C43859%2C1732402385760.1732402409146
2024-11-23T22:53:45,121 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,177 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,177 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-23T22:53:45,291 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:53:45,291 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:53:45,292 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:45,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:45,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:45,293 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:53:45,293 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:53:45,293 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=401360406, stopped=false
2024-11-23T22:53:45,293 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,45639,1732402385336
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:45,376 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:53:45,377 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:53:45,378 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:45,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:45,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:45,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:45,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:45,380 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,43859,1732402385760' *****
2024-11-23T22:53:45,380 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:53:45,380 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,36341,1732402387015' *****
2024-11-23T22:53:45,380 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:53:45,380 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:53:45,380 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:53:45,380 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:53:45,381 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:53:45,381 INFO  [RS:1;dd96e8dc8369:36341 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(3091): Received CLOSE for 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:45,381 INFO  [RS:1;dd96e8dc8369:36341 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:53:45,381 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:45,381 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:53:45,381 INFO  [RS:1;dd96e8dc8369:36341 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd96e8dc8369:36341.
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:53:45,381 DEBUG [RS:1;dd96e8dc8369:36341 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:45,381 DEBUG [RS:1;dd96e8dc8369:36341 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:45,381 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 377d9f4bf35a8f2c0bbf065ae6c0d147, disabling compactions & flushes
2024-11-23T22:53:45,381 INFO  [RS:0;dd96e8dc8369:43859 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:43859.
2024-11-23T22:53:45,382 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:45,382 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,36341,1732402387015; all regions closed.
2024-11-23T22:53:45,382 DEBUG [RS:0;dd96e8dc8369:43859 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:53:45,382 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:45,382 DEBUG [RS:0;dd96e8dc8369:43859 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:45,382 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147. after waiting 0 ms
2024-11-23T22:53:45,382 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:45,382 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:53:45,382 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:53:45,382 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:53:45,382 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:53:45,382 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:53:45,382 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,382 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-23T22:53:45,382 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 377d9f4bf35a8f2c0bbf065ae6c0d147=TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.}
2024-11-23T22:53:45,383 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,383 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:53:45,383 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,383 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:53:45,383 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,383 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:53:45,383 DEBUG [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 377d9f4bf35a8f2c0bbf065ae6c0d147
2024-11-23T22:53:45,383 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:53:45,383 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,383 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:53:45,383 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-23T22:53:45,383 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,383 ERROR [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c-prefix:dd96e8dc8369,43859,1732402385760.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,384 WARN  [FSHLog-0-hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c-prefix:dd96e8dc8369,43859,1732402385760.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,384 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,384 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
2024-11-23T22:53:45,384 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C43859%2C1732402385760.meta:.meta(num 1732402386791) roll requested
2024-11-23T22:53:45,384 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C43859%2C1732402385760.meta.1732402425384.meta
2024-11-23T22:53:45,384 WARN  [IPC Server handler 1 on default port 39715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741837_1013
2024-11-23T22:53:45,385 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 after 1ms
2024-11-23T22:53:45,388 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/20abbf78c4a34eae9272d71535949f19 is 1079, key is tmprow/info:/1732402425089/Put/seqid=0
2024-11-23T22:53:45,390 WARN  [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1081
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:48598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10]'}, localName='127.0.0.1:35683', datanodeUuid='f662443b-02eb-4dfb-847c-b712769e12a5', xmitsInProgress=0}:Exception transferring block BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081 to mirror 127.0.0.1:35033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:45,390 WARN  [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-69e4986e-7e81-4236-a28c-7fdb8717ab30,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:45,390 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:48598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081] {}] datanode.BlockReceiver(316): Block 1073741896 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-23T22:53:45,390 WARN  [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081
2024-11-23T22:53:45,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1110332203_22 at /127.0.0.1:48598 [Receiving block BP-942195297-172.17.0.2-1732402382938:blk_1073741896_1081] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48598 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:45,391 WARN  [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:45,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741897_1082 (size=6027)
2024-11-23T22:53:45,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741897_1082 (size=6027)
2024-11-23T22:53:45,395 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/20abbf78c4a34eae9272d71535949f19
2024-11-23T22:53:45,401 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,401 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,402 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,402 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,402 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/.tmp/info/20abbf78c4a34eae9272d71535949f19 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/20abbf78c4a34eae9272d71535949f19
2024-11-23T22:53:45,402 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,402 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402425384.meta
2024-11-23T22:53:45,402 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,403 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38865,DS-fa6dfd85-1b6a-4723-8208-3f3302fdad66,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,403 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
2024-11-23T22:53:45,403 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39203:39203),(127.0.0.1/127.0.0.1:39717:39717)]
2024-11-23T22:53:45,403 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta is not closed yet, will try archiving it next time
2024-11-23T22:53:45,403 WARN  [IPC Server handler 2 on default port 39715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta has not been closed. Lease recovery is in progress. RecoveryId = 1084 for block blk_1073741834_1016
2024-11-23T22:53:45,403 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta after 0ms
2024-11-23T22:53:45,408 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/20abbf78c4a34eae9272d71535949f19, entries=1, sequenceid=83, filesize=5.9 K
2024-11-23T22:53:45,410 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 27ms, sequenceid=83, compaction requested=false
2024-11-23T22:53:45,410 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21] to archive
2024-11-23T22:53:45,411 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:53:45,413 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/92faaa153ecd4e32ae5cf18561c99219
2024-11-23T22:53:45,415 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/aa789228bd2f483e8cbfacd978ad0626
2024-11-23T22:53:45,416 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/45412edee662459795feca520fa126ed
2024-11-23T22:53:45,417 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/2b81e68a8d2f4195ac9a8187002efb85
2024-11-23T22:53:45,418 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/4c57adc4846b459dbdb6fe75164fbdaa
2024-11-23T22:53:45,419 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8df82d9f29c348969819fc35c3ddcfd4
2024-11-23T22:53:45,420 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dfbced2edeb2417b909ac9b3c602e92a
2024-11-23T22:53:45,421 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/8196ef38551d405a9e188d1003f3a146
2024-11-23T22:53:45,423 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21 to hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/info/dd90066588b04e96b66bdd7f7ddc7b21
2024-11-23T22:53:45,423 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/info/ea50a3196ebd4dd39cc442caecb50d46 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147./info:regioninfo/1732402387526/Put/seqid=0
2024-11-23T22:53:45,423 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=dd96e8dc8369:45639 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-23T22:53:45,424 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [92faaa153ecd4e32ae5cf18561c99219=10347, aa789228bd2f483e8cbfacd978ad0626=12506, 45412edee662459795feca520fa126ed=17994, 2b81e68a8d2f4195ac9a8187002efb85=6027, 4c57adc4846b459dbdb6fe75164fbdaa=6027, 8df82d9f29c348969819fc35c3ddcfd4=18097, dfbced2edeb2417b909ac9b3c602e92a=6027, 8196ef38551d405a9e188d1003f3a146=9267, dd90066588b04e96b66bdd7f7ddc7b21=13583]
2024-11-23T22:53:45,433 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/377d9f4bf35a8f2c0bbf065ae6c0d147/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1
2024-11-23T22:53:45,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741899_1085 (size=7089)
2024-11-23T22:53:45,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741899_1085 (size=7089)
2024-11-23T22:53:45,434 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:45,434 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 377d9f4bf35a8f2c0bbf065ae6c0d147:
Waiting for close lock at 1732402425381Running coprocessor pre-close hooks at 1732402425381Disabling compacts and flushes for region at 1732402425381Disabling writes for close at 1732402425382 (+1 ms)Obtaining lock to block concurrent updates at 1732402425382Preparing flush snapshotting stores in 377d9f4bf35a8f2c0bbf065ae6c0d147 at 1732402425382Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732402425383 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147. at 1732402425383Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147/info: creating writer at 1732402425383Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147/info: appending metadata at 1732402425387 (+4 ms)Flushing 377d9f4bf35a8f2c0bbf065ae6c0d147/info: closing flushed file at 1732402425387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289358f2: reopening flushed file at 1732402425401 (+14 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 377d9f4bf35a8f2c0bbf065ae6c0d147 in 27ms, sequenceid=83, compaction requested=false at 1732402425410 (+9 ms)Writing region close event to WAL at 1732402425425 (+15 ms)Running coprocessor post-close hooks at 1732402425434 (+9 ms)Closed at 1732402425434
2024-11-23T22:53:45,434 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732402387148.377d9f4bf35a8f2c0bbf065ae6c0d147.
2024-11-23T22:53:45,435 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/info/ea50a3196ebd4dd39cc442caecb50d46
2024-11-23T22:53:45,453 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/ns/a21642e9fd4f498fa53ccadb1607ff10 is 43, key is default/ns:d/1732402386873/Put/seqid=0
2024-11-23T22:53:45,455 WARN  [Thread-1072 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:53:45,455 WARN  [Thread-1072 {}] hdfs.DataStreamer(1731): Error Recovery for BP-942195297-172.17.0.2-1732402382938:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK], DatanodeInfoWithStorage[127.0.0.1:45915,DS-5c04e7e2-d6cd-4af7-8ddf-4dea8eeb7591,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]) is bad.
2024-11-23T22:53:45,455 WARN  [Thread-1072 {}] hdfs.DataStreamer(1850): Abandoning BP-942195297-172.17.0.2-1732402382938:blk_1073741900_1086
2024-11-23T22:53:45,456 WARN  [Thread-1072 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35033,DS-ac7b0262-4c40-4940-9081-592113fa766e,DISK]
2024-11-23T22:53:45,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741901_1087 (size=5153)
2024-11-23T22:53:45,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741901_1087 (size=5153)
2024-11-23T22:53:45,461 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/ns/a21642e9fd4f498fa53ccadb1607ff10
2024-11-23T22:53:45,482 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/table/add7162036474b0bbcfc905feef405fd is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732402387540/Put/seqid=0
2024-11-23T22:53:45,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741902_1088 (size=5424)
2024-11-23T22:53:45,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741902_1088 (size=5424)
2024-11-23T22:53:45,488 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/table/add7162036474b0bbcfc905feef405fd
2024-11-23T22:53:45,494 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/info/ea50a3196ebd4dd39cc442caecb50d46 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/info/ea50a3196ebd4dd39cc442caecb50d46
2024-11-23T22:53:45,501 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/info/ea50a3196ebd4dd39cc442caecb50d46, entries=10, sequenceid=11, filesize=6.9 K
2024-11-23T22:53:45,502 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/ns/a21642e9fd4f498fa53ccadb1607ff10 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/ns/a21642e9fd4f498fa53ccadb1607ff10
2024-11-23T22:53:45,509 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/ns/a21642e9fd4f498fa53ccadb1607ff10, entries=2, sequenceid=11, filesize=5.0 K
2024-11-23T22:53:45,510 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/.tmp/table/add7162036474b0bbcfc905feef405fd as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/table/add7162036474b0bbcfc905feef405fd
2024-11-23T22:53:45,517 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/table/add7162036474b0bbcfc905feef405fd, entries=2, sequenceid=11, filesize=5.3 K
2024-11-23T22:53:45,518 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false
2024-11-23T22:53:45,527 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-23T22:53:45,527 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:53:45,528 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:45,528 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402425383Running coprocessor pre-close hooks at 1732402425383Disabling compacts and flushes for region at 1732402425383Disabling writes for close at 1732402425383Obtaining lock to block concurrent updates at 1732402425383Preparing flush snapshotting stores in 1588230740 at 1732402425383Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732402425384 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732402425403 (+19 ms)Flushing 1588230740/info: creating writer at 1732402425404 (+1 ms)Flushing 1588230740/info: appending metadata at 1732402425423 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732402425423Flushing 1588230740/ns: creating writer at 1732402425440 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732402425453 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732402425453Flushing 1588230740/table: creating writer at 1732402425467 (+14 ms)Flushing 1588230740/table: appending metadata at 1732402425481 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732402425481Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54b5e3ac: reopening flushed file at 1732402425493 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1657d1bc: reopening flushed file at 1732402425501 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@637849b4: reopening flushed file at 1732402425509 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false at 1732402425518 (+9 ms)Writing region close event to WAL at 1732402425523 (+5 ms)Running coprocessor post-close hooks at 1732402425527 (+4 ms)Closed at 1732402425528 (+1 ms)
2024-11-23T22:53:45,528 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:45,583 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,43859,1732402385760; all regions closed.
2024-11-23T22:53:45,584 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,584 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,584 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,584 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,584 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:45,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741898_1083 (size=825)
2024-11-23T22:53:45,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741898_1083 (size=825)
2024-11-23T22:53:46,117 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-23T22:53:46,117 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-23T22:53:46,252 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-23T22:53:46,279 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-23T22:53:46,281 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:46,838 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@759272bf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45915, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=39203, infoSecurePort=0, ipcPort=42363, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741831_1007 to 127.0.0.1:35033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:46,852 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3972a23c {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-942195297-172.17.0.2-1732402382938:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:38865,null,null])
java.net.ConnectException: Call From dd96e8dc8369/172.17.0.2 to localhost:38719 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-23T22:53:46,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741830_1073 (size=27282)
2024-11-23T22:53:46,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741835_1011 (size=393)
2024-11-23T22:53:47,118 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:47,658 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-23T22:53:47,658 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-23T22:53:47,838 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@8019efa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45915, datanodeUuid=969d3a7d-7768-473e-9942-01ee0825f8da, infoPort=39203, infoSecurePort=0, ipcPort=42363, storageInfo=lv=-57;cid=testClusterID;nsid=1711752669;c=1732402382938):Failed to transfer BP-942195297-172.17.0.2-1732402382938:blk_1073741829_1005 to 127.0.0.1:35033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:53:47,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:48,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:48,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:48,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741876_1059 (size=13268)
2024-11-23T22:53:49,386 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 after 4002ms
2024-11-23T22:53:49,405 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta after 4002ms
2024-11-23T22:53:49,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:50,384 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-23T22:53:50,386 DEBUG [RS:1;dd96e8dc8369:36341 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs
2024-11-23T22:53:50,386 INFO  [RS:1;dd96e8dc8369:36341 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C36341%2C1732402387015:(num 1732402387251)
2024-11-23T22:53:50,386 DEBUG [RS:1;dd96e8dc8369:36341 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:50,386 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:50,386 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:53:50,387 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:53:50,387 INFO  [RS:1;dd96e8dc8369:36341 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36341
2024-11-23T22:53:50,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:50,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,469 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:50,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,36341,1732402387015
2024-11-23T22:53:50,475 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:53:50,585 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-23T22:53:50,591 DEBUG [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs
2024-11-23T22:53:50,591 INFO  [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C43859%2C1732402385760.meta:.meta(num 1732402425384)
2024-11-23T22:53:50,592 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:50,592 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:50,592 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:50,592 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:50,593 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:50,594 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,36341,1732402387015]
2024-11-23T22:53:50,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741891_1075 (size=18156)
2024-11-23T22:53:50,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741891_1075 (size=18156)
2024-11-23T22:53:50,667 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,36341,1732402387015 already deleted, retry=false
2024-11-23T22:53:50,668 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,36341,1732402387015 expired; onlineServers=1
2024-11-23T22:53:50,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:50,694 INFO  [RS:1;dd96e8dc8369:36341 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:53:50,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36341-0x10169e68d520002, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:50,694 INFO  [RS:1;dd96e8dc8369:36341 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,36341,1732402387015; zookeeper connection closed.
2024-11-23T22:53:50,695 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2af6090d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2af6090d
2024-11-23T22:53:50,971 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:53:50,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:50,999 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:51,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:51,404 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-23T22:53:51,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:51,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741830_1073 (size=27282)
2024-11-23T22:53:51,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-23T22:53:51,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:53:51,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:53:52,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:52,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:53,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:53,404 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-23T22:53:53,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:54,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:54,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:54,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:54,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741836_1012 (size=76)
2024-11-23T22:53:55,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:55,405 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-23T22:53:55,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:55,593 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-23T22:53:55,599 DEBUG [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/oldWALs
2024-11-23T22:53:55,599 INFO  [RS:0;dd96e8dc8369:43859 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C43859%2C1732402385760:(num 1732402424640)
2024-11-23T22:53:55,599 DEBUG [RS:0;dd96e8dc8369:43859 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:53:55,599 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:53:55,599 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:53:55,599 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:53:55,600 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:53:55,600 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:53:55,600 INFO  [RS:0;dd96e8dc8369:43859 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43859
2024-11-23T22:53:55,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,43859,1732402385760
2024-11-23T22:53:55,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:55,669 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:53:55,680 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,43859,1732402385760]
2024-11-23T22:53:55,690 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,43859,1732402385760 already deleted, retry=false
2024-11-23T22:53:55,690 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,43859,1732402385760 expired; onlineServers=0
2024-11-23T22:53:55,690 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,45639,1732402385336' *****
2024-11-23T22:53:55,690 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:53:55,690 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:53:55,690 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:53:55,690 DEBUG [M:0;dd96e8dc8369:45639 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:53:55,690 DEBUG [M:0;dd96e8dc8369:45639 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:53:55,690 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:53:55,690 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402386154 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402386154,5,FailOnTimeoutGroup]
2024-11-23T22:53:55,690 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402386154 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402386154,5,FailOnTimeoutGroup]
2024-11-23T22:53:55,691 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:53:55,691 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:53:55,691 DEBUG [M:0;dd96e8dc8369:45639 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:53:55,691 INFO  [M:0;dd96e8dc8369:45639 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:53:55,691 INFO  [M:0;dd96e8dc8369:45639 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:53:55,691 INFO  [M:0;dd96e8dc8369:45639 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:53:55,691 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:53:55,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:53:55,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:55,700 DEBUG [M:0;dd96e8dc8369:45639 {}] zookeeper.ZKUtil(347): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:53:55,700 WARN  [M:0;dd96e8dc8369:45639 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:53:55,701 INFO  [M:0;dd96e8dc8369:45639 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/.lastflushedseqids
2024-11-23T22:53:55,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741903_1089 (size=130)
2024-11-23T22:53:55,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741903_1089 (size=130)
2024-11-23T22:53:55,706 INFO  [M:0;dd96e8dc8369:45639 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:53:55,707 INFO  [M:0;dd96e8dc8369:45639 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:53:55,707 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:53:55,707 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:55,707 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:55,707 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:53:55,707 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:55,707 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.22 KB heapSize=29.46 KB
2024-11-23T22:53:55,724 DEBUG [M:0;dd96e8dc8369:45639 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05eaa1835e834ec5b0c2d6e5b931013e is 82, key is hbase:meta,,1/info:regioninfo/1732402386823/Put/seqid=0
2024-11-23T22:53:55,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741904_1090 (size=5672)
2024-11-23T22:53:55,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741904_1090 (size=5672)
2024-11-23T22:53:55,729 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05eaa1835e834ec5b0c2d6e5b931013e
2024-11-23T22:53:55,754 DEBUG [M:0;dd96e8dc8369:45639 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d197ede95084e43b659575c927f5c4f is 772, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402387545/Put/seqid=0
2024-11-23T22:53:55,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741905_1091 (size=6253)
2024-11-23T22:53:55,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741905_1091 (size=6253)
2024-11-23T22:53:55,760 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.56 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d197ede95084e43b659575c927f5c4f
2024-11-23T22:53:55,765 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6d197ede95084e43b659575c927f5c4f
2024-11-23T22:53:55,778 DEBUG [M:0;dd96e8dc8369:45639 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9386833adb34d18bafa401737906c8e is 69, key is dd96e8dc8369,36341,1732402387015/rs:state/1732402387092/Put/seqid=0
2024-11-23T22:53:55,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:55,780 INFO  [RS:0;dd96e8dc8369:43859 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:53:55,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x10169e68d520001, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:55,780 INFO  [RS:0;dd96e8dc8369:43859 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,43859,1732402385760; zookeeper connection closed.
2024-11-23T22:53:55,780 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52141cc1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52141cc1
2024-11-23T22:53:55,781 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-23T22:53:55,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741906_1092 (size=5224)
2024-11-23T22:53:55,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741906_1092 (size=5224)
2024-11-23T22:53:55,784 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9386833adb34d18bafa401737906c8e
2024-11-23T22:53:55,801 DEBUG [M:0;dd96e8dc8369:45639 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e344f30db9bf472c8a2ebecc8a4b37c8 is 52, key is load_balancer_on/state:d/1732402386998/Put/seqid=0
2024-11-23T22:53:55,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741907_1093 (size=5056)
2024-11-23T22:53:55,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741907_1093 (size=5056)
2024-11-23T22:53:55,806 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e344f30db9bf472c8a2ebecc8a4b37c8
2024-11-23T22:53:55,811 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05eaa1835e834ec5b0c2d6e5b931013e as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05eaa1835e834ec5b0c2d6e5b931013e
2024-11-23T22:53:55,817 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05eaa1835e834ec5b0c2d6e5b931013e, entries=8, sequenceid=60, filesize=5.5 K
2024-11-23T22:53:55,818 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d197ede95084e43b659575c927f5c4f as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6d197ede95084e43b659575c927f5c4f
2024-11-23T22:53:55,823 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6d197ede95084e43b659575c927f5c4f
2024-11-23T22:53:55,823 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6d197ede95084e43b659575c927f5c4f, entries=6, sequenceid=60, filesize=6.1 K
2024-11-23T22:53:55,824 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9386833adb34d18bafa401737906c8e as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e9386833adb34d18bafa401737906c8e
2024-11-23T22:53:55,830 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e9386833adb34d18bafa401737906c8e, entries=2, sequenceid=60, filesize=5.1 K
2024-11-23T22:53:55,831 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e344f30db9bf472c8a2ebecc8a4b37c8 as hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e344f30db9bf472c8a2ebecc8a4b37c8
2024-11-23T22:53:55,837 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e344f30db9bf472c8a2ebecc8a4b37c8, entries=1, sequenceid=60, filesize=4.9 K
2024-11-23T22:53:55,838 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.22 KB/23781, heapSize ~29.40 KB/30104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=60, compaction requested=false
2024-11-23T22:53:55,839 INFO  [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:55,839 DEBUG [M:0;dd96e8dc8369:45639 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402435707Disabling compacts and flushes for region at 1732402435707Disabling writes for close at 1732402435707Obtaining lock to block concurrent updates at 1732402435707Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402435707Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23781, getHeapSize=30104, getOffHeapSize=0, getCellsCount=71 at 1732402435707Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402435708 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402435708Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402435724 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402435724Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402435735 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402435754 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402435754Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402435765 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402435778 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402435778Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402435788 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402435800 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402435800Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e90cfc6: reopening flushed file at 1732402435810 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e991b5f: reopening flushed file at 1732402435817 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23b5db5: reopening flushed file at 1732402435823 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61412a15: reopening flushed file at 1732402435830 (+7 ms)Finished flush of dataSize ~23.22 KB/23781, heapSize ~29.40 KB/30104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=60, compaction requested=false at 1732402435838 (+8 ms)Writing region close event to WAL at 1732402435839 (+1 ms)Closed at 1732402435839
2024-11-23T22:53:55,840 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:55,840 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:55,840 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:55,840 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:55,840 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:53:55,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741889_1072 (size=1045)
2024-11-23T22:53:55,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45915 is added to blk_1073741889_1072 (size=1045)
2024-11-23T22:53:55,843 INFO  [M:0;dd96e8dc8369:45639 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:53:55,843 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:53:55,843 INFO  [M:0;dd96e8dc8369:45639 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45639
2024-11-23T22:53:55,843 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:53:56,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:56,031 INFO  [M:0;dd96e8dc8369:45639 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:53:56,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45639-0x10169e68d520000, quorum=127.0.0.1:62169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:53:56,036 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3af3a5c0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:56,037 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@67e24eab{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:56,038 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:56,038 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ef1ec46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:56,038 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@44bf135d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:56,041 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:56,041 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid 969d3a7d-7768-473e-9942-01ee0825f8da) service to localhost/127.0.0.1:39715
2024-11-23T22:53:56,041 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:56,041 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:56,041 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d0b10cc {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:38865,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:38719 , LocalHost:localPort dd96e8dc8369/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-23T22:53:56,041 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d0b10cc {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-942195297-172.17.0.2-1732402382938:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45915,null,null], DatanodeInfoWithStorage[127.0.0.1:38865,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-942195297-172.17.0.2-1732402382938
2024-11-23T22:53:56,042 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data3/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:56,042 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data4/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:56,043 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:56,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20e9d233{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:56,049 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7072e3f6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:56,049 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:56,049 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a60480b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:56,049 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dfb8150{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:56,051 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:53:56,051 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:53:56,051 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:53:56,051 WARN  [BP-942195297-172.17.0.2-1732402382938 heartbeating to localhost/127.0.0.1:39715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-942195297-172.17.0.2-1732402382938 (Datanode Uuid f662443b-02eb-4dfb-847c-b712769e12a5) service to localhost/127.0.0.1:39715
2024-11-23T22:53:56,052 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data9/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:56,052 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/cluster_7ad71e70-6a94-508e-5ac4-ad79c13c927e/data/data10/current/BP-942195297-172.17.0.2-1732402382938 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:53:56,052 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:53:56,057 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6880627a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:53:56,057 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d3a3315{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:53:56,057 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:53:56,057 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@281d64b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:53:56,057 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41fecd82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir/,STOPPED}
2024-11-23T22:53:56,064 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:53:56,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:53:56,105 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=158 (was 82)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39715
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39715
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39715
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39715 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39715
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35407
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39715 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39715 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:39715
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:39715
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f413cbf46e8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f413cbf46e8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f413cbf46e8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39715 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35407
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39715
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=438 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=151 (was 171), ProcessCount=11 (was 11), AvailableMemoryMB=6801 (was 7875)
2024-11-23T22:53:56,112 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=158, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=151, ProcessCount=11, AvailableMemoryMB=6801
2024-11-23T22:53:56,112 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.log.dir so I do NOT create it in target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9fba29e-ebfa-57b8-d78f-c2b9342d1110/hadoop.tmp.dir so I do NOT create it in target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09, deleteOnExit=true
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/test.cache.data in system properties and HBase conf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:53:56,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:53:56,113 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:53:56,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:53:56,126 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:53:56,137 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:56,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:56,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:56,491 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:56,496 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:56,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:56,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:56,497 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:53:56,498 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:56,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2720e386{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:56,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73f89830{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:56,590 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@318626e1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-35421-hadoop-hdfs-3_4_1-tests_jar-_-any-11379409094222415866/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:53:56,591 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11ec3e38{HTTP/1.1, (http/1.1)}{localhost:35421}
2024-11-23T22:53:56,591 INFO  [Time-limited test {}] server.Server(415): Started @161916ms
2024-11-23T22:53:56,602 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:53:56,815 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:56,818 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:56,818 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:56,818 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:56,818 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:56,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2eb03db7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:56,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48e9bd26{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:56,916 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@245efd98{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-44599-hadoop-hdfs-3_4_1-tests_jar-_-any-271048936201294766/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:56,916 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@200ef94f{HTTP/1.1, (http/1.1)}{localhost:44599}
2024-11-23T22:53:56,917 INFO  [Time-limited test {}] server.Server(415): Started @162242ms
2024-11-23T22:53:56,919 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:56,959 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:53:56,974 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,974 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,974 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,974 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,975 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,975 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,978 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,979 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,979 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,981 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:53:56,988 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:53:56,992 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:53:56,996 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:53:56,996 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:53:56,996 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:53:56,997 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32ad5d3f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:53:56,997 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58716200{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:53:57,092 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b7c275b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-41325-hadoop-hdfs-3_4_1-tests_jar-_-any-13018293534845363102/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:53:57,093 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7ca0c1f{HTTP/1.1, (http/1.1)}{localhost:41325}
2024-11-23T22:53:57,093 INFO  [Time-limited test {}] server.Server(415): Started @162418ms
2024-11-23T22:53:57,094 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:53:57,138 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:57,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:57,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:58,139 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:58,308 WARN  [Thread-1221 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data1/current/BP-1031033842-172.17.0.2-1732402436130/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:58,308 WARN  [Thread-1222 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data2/current/BP-1031033842-172.17.0.2-1732402436130/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:58,332 WARN  [Thread-1184 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:58,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf85e86597b8523c with lease ID 0x1497b22ae0d82f06: Processing first storage report for DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff from datanode DatanodeRegistration(127.0.0.1:35405, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=39499, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130)
2024-11-23T22:53:58,335 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf85e86597b8523c with lease ID 0x1497b22ae0d82f06: from storage DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff node DatanodeRegistration(127.0.0.1:35405, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=39499, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:58,335 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf85e86597b8523c with lease ID 0x1497b22ae0d82f06: Processing first storage report for DS-6a21eede-6947-45f8-b0ef-dc3590a9a56a from datanode DatanodeRegistration(127.0.0.1:35405, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=39499, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130)
2024-11-23T22:53:58,335 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf85e86597b8523c with lease ID 0x1497b22ae0d82f06: from storage DS-6a21eede-6947-45f8-b0ef-dc3590a9a56a node DatanodeRegistration(127.0.0.1:35405, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=39499, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:58,401 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:58,411 WARN  [Thread-1232 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data3/current/BP-1031033842-172.17.0.2-1732402436130/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:58,411 WARN  [Thread-1233 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data4/current/BP-1031033842-172.17.0.2-1732402436130/current, will proceed with Du for space computation calculation, 
2024-11-23T22:53:58,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:58,436 WARN  [Thread-1208 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:53:58,438 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x906f7995eff81b5 with lease ID 0x1497b22ae0d82f07: Processing first storage report for DS-d8aa46c0-bbf1-4e21-b773-12b31d983533 from datanode DatanodeRegistration(127.0.0.1:41523, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=33625, infoSecurePort=0, ipcPort=33645, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130)
2024-11-23T22:53:58,438 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x906f7995eff81b5 with lease ID 0x1497b22ae0d82f07: from storage DS-d8aa46c0-bbf1-4e21-b773-12b31d983533 node DatanodeRegistration(127.0.0.1:41523, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=33625, infoSecurePort=0, ipcPort=33645, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:58,438 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x906f7995eff81b5 with lease ID 0x1497b22ae0d82f07: Processing first storage report for DS-f8491553-8210-4e5f-b182-ac4dc48c97ae from datanode DatanodeRegistration(127.0.0.1:41523, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=33625, infoSecurePort=0, ipcPort=33645, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130)
2024-11-23T22:53:58,438 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x906f7995eff81b5 with lease ID 0x1497b22ae0d82f07: from storage DS-f8491553-8210-4e5f-b182-ac4dc48c97ae node DatanodeRegistration(127.0.0.1:41523, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=33625, infoSecurePort=0, ipcPort=33645, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:53:58,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf
2024-11-23T22:53:58,536 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/zookeeper_0, clientPort=63019, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:53:58,537 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63019
2024-11-23T22:53:58,537 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,539 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:58,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:53:58,552 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc with version=8
2024-11-23T22:53:58,552 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:53:58,555 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:58,556 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46645
2024-11-23T22:53:58,558 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46645 connecting to ZooKeeper ensemble=127.0.0.1:63019
2024-11-23T22:53:58,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466450x0, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:58,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46645-0x10169e75d320000 connected
2024-11-23T22:53:58,728 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,729 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,732 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:58,732 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc, hbase.cluster.distributed=false
2024-11-23T22:53:58,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:58,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46645
2024-11-23T22:53:58,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46645
2024-11-23T22:53:58,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46645
2024-11-23T22:53:58,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46645
2024-11-23T22:53:58,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46645
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:53:58,754 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:53:58,755 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:53:58,755 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40259
2024-11-23T22:53:58,757 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40259 connecting to ZooKeeper ensemble=127.0.0.1:63019
2024-11-23T22:53:58,757 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,759 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,769 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402590x0, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:53:58,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40259-0x10169e75d320001 connected
2024-11-23T22:53:58,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:53:58,770 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:53:58,771 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:53:58,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:53:58,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:53:58,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40259
2024-11-23T22:53:58,775 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40259
2024-11-23T22:53:58,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40259
2024-11-23T22:53:58,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40259
2024-11-23T22:53:58,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40259
2024-11-23T22:53:58,789 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:46645
2024-11-23T22:53:58,790 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:58,801 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:58,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:58,802 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:58,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,811 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:53:58,812 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,812 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:53:58,812 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,46645,1732402438555 from backup master directory
2024-11-23T22:53:58,822 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:58,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:58,822 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:58,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:53:58,822 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:58,826 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/hbase.id] with ID: 3894aea4-cd09-4ca0-b0cc-9b1159a1bd56
2024-11-23T22:53:58,826 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/.tmp/hbase.id
2024-11-23T22:53:58,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:58,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:53:58,832 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/.tmp/hbase.id]:[hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/hbase.id]
2024-11-23T22:53:58,843 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:53:58,843 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:53:58,844 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-23T22:53:58,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,854 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:58,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:53:58,860 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:53:58,861 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:53:58,861 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:58,867 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:58,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:53:58,868 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store
2024-11-23T22:53:58,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:58,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:53:58,875 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:58,875 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:53:58,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402438875Disabling compacts and flushes for region at 1732402438875Disabling writes for close at 1732402438875Writing region close event to WAL at 1732402438875Closed at 1732402438875
2024-11-23T22:53:58,876 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/.initializing
2024-11-23T22:53:58,876 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:58,878 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C46645%2C1732402438555, suffix=, logDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555, archiveDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/oldWALs, maxLogs=10
2024-11-23T22:53:58,879 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46645%2C1732402438555.1732402438879
2024-11-23T22:53:58,884 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879
2024-11-23T22:53:58,888 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39499:39499),(127.0.0.1/127.0.0.1:33625:33625)]
2024-11-23T22:53:58,889 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:58,889 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:58,889 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,889 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:58,892 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:53:58,892 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:58,893 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:58,893 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:58,894 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:53:58,894 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:58,894 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:58,895 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:58,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:53:58,896 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:58,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:58,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:53:58,897 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:53:58,897 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:58,898 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:53:58,898 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,898 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,898 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,899 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,899 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,900 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:53:58,901 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:53:58,904 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:58,905 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708570, jitterRate=-0.09900672733783722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:53:58,905 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402438890Initializing all the Stores at 1732402438891 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402438891Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402438891Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402438891Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402438891Cleaning up temporary data from old regions at 1732402438899 (+8 ms)Region opened successfully at 1732402438905 (+6 ms)
2024-11-23T22:53:58,906 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:53:58,908 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3231c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:58,909 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:53:58,909 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:53:58,909 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:53:58,910 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:53:58,910 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:53:58,910 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:53:58,910 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:53:58,913 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:53:58,913 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:53:58,923 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:53:58,924 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:53:58,925 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:53:58,934 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:53:58,934 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:53:58,936 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:53:58,949 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:53:58,950 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:53:58,959 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:53:58,962 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:53:58,969 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:53:58,980 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:58,980 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:53:58,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:58,982 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,46645,1732402438555, sessionid=0x10169e75d320000, setting cluster-up flag (Was=false)
2024-11-23T22:53:59,001 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,033 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:53:59,033 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:59,054 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,085 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:53:59,087 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,46645,1732402438555
2024-11-23T22:53:59,089 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:53:59,091 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:59,091 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:53:59,091 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:53:59,092 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,46645,1732402438555
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:53:59,094 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:59,094 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:59,094 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:59,095 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:53:59,095 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:53:59,095 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,095 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:59,095 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,098 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:59,098 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:53:59,099 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402469099
2024-11-23T22:53:59,099 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:53:59,100 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:53:59,100 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:53:59,100 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:53:59,100 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:53:59,100 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:53:59,100 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,100 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:53:59,102 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,102 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:53:59,102 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:53:59,102 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:53:59,103 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:53:59,103 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:53:59,103 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402439103,5,FailOnTimeoutGroup]
2024-11-23T22:53:59,104 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402439103,5,FailOnTimeoutGroup]
2024-11-23T22:53:59,104 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,104 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:53:59,104 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,104 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:59,114 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:53:59,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:53:59,114 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc
2024-11-23T22:53:59,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:59,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:53:59,139 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:59,182 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(746): ClusterId : 3894aea4-cd09-4ca0-b0cc-9b1159a1bd56
2024-11-23T22:53:59,182 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:53:59,198 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:53:59,198 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:53:59,209 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:53:59,209 DEBUG [RS:0;dd96e8dc8369:40259 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767dbdb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:53:59,224 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:40259
2024-11-23T22:53:59,224 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:53:59,224 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:53:59,224 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:53:59,225 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,46645,1732402438555 with port=40259, startcode=1732402438754
2024-11-23T22:53:59,225 DEBUG [RS:0;dd96e8dc8369:40259 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:53:59,227 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40427, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:53:59,228 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46645 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,228 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46645 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,230 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc
2024-11-23T22:53:59,230 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41217
2024-11-23T22:53:59,230 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:53:59,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:53:59,240 DEBUG [RS:0;dd96e8dc8369:40259 {}] zookeeper.ZKUtil(111): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,240 WARN  [RS:0;dd96e8dc8369:40259 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:53:59,240 INFO  [RS:0;dd96e8dc8369:40259 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:59,240 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,240 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,40259,1732402438754]
2024-11-23T22:53:59,243 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:53:59,245 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:53:59,245 INFO  [RS:0;dd96e8dc8369:40259 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:53:59,245 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,245 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:53:59,246 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:53:59,246 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,246 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,246 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,246 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:59,247 DEBUG [RS:0;dd96e8dc8369:40259 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:53:59,251 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,252 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,252 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,252 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,252 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,252 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,40259,1732402438754-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:53:59,266 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:53:59,267 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,40259,1732402438754-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,267 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,267 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.Replication(171): dd96e8dc8369,40259,1732402438754 started
2024-11-23T22:53:59,281 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:53:59,282 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,40259,1732402438754, RpcServer on dd96e8dc8369/172.17.0.2:40259, sessionid=0x10169e75d320001
2024-11-23T22:53:59,282 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:53:59,282 DEBUG [RS:0;dd96e8dc8369:40259 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,282 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,40259,1732402438754'
2024-11-23T22:53:59,282 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:53:59,282 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,40259,1732402438754'
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:53:59,283 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:53:59,284 DEBUG [RS:0;dd96e8dc8369:40259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:53:59,284 INFO  [RS:0;dd96e8dc8369:40259 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:53:59,284 INFO  [RS:0;dd96e8dc8369:40259 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:53:59,386 INFO  [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C40259%2C1732402438754, suffix=, logDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754, archiveDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs, maxLogs=32
2024-11-23T22:53:59,387 INFO  [RS:0;dd96e8dc8369:40259 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:53:59,392 INFO  [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:53:59,396 DEBUG [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39499:39499),(127.0.0.1/127.0.0.1:33625:33625)]
2024-11-23T22:53:59,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:59,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:53:59,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:59,530 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:59,533 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:59,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,534 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,535 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:59,537 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:59,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:59,541 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:59,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,542 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,542 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:59,544 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:59,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,545 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:59,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740
2024-11-23T22:53:59,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740
2024-11-23T22:53:59,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:59,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:59,549 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:59,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:59,552 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:53:59,552 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=830425, jitterRate=0.05594000220298767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:59,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402439528Initializing all the Stores at 1732402439529 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439529Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439530 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402439530Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439530Cleaning up temporary data from old regions at 1732402439548 (+18 ms)Region opened successfully at 1732402439553 (+5 ms)
2024-11-23T22:53:59,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:53:59,553 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:53:59,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:53:59,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:53:59,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:53:59,554 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:53:59,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402439553Disabling compacts and flushes for region at 1732402439553Disabling writes for close at 1732402439553Writing region close event to WAL at 1732402439554 (+1 ms)Closed at 1732402439554
2024-11-23T22:53:59,555 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:53:59,555 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:53:59,555 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:53:59,556 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:59,557 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:53:59,707 DEBUG [dd96e8dc8369:46645 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:53:59,708 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,710 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,40259,1732402438754, state=OPENING
2024-11-23T22:53:59,759 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:53:59,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,769 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:53:59,770 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:53:59,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:59,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:53:59,770 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40259,1732402438754}]
2024-11-23T22:53:59,923 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:53:59,925 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:53:59,929 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:53:59,929 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:53:59,931 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C40259%2C1732402438754.meta, suffix=.meta, logDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754, archiveDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs, maxLogs=32
2024-11-23T22:53:59,932 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta
2024-11-23T22:53:59,938 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta
2024-11-23T22:53:59,939 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39499:39499),(127.0.0.1/127.0.0.1:33625:33625)]
2024-11-23T22:53:59,940 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:53:59,940 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:53:59,941 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:53:59,941 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:53:59,941 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:53:59,941 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:53:59,941 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:53:59,941 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:53:59,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:53:59,949 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:53:59,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:53:59,951 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:53:59,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,952 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:53:59,952 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:53:59,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:53:59,953 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:53:59,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:53:59,954 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:53:59,954 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:53:59,955 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740
2024-11-23T22:53:59,956 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740
2024-11-23T22:53:59,957 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:53:59,957 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:53:59,958 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:53:59,959 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:53:59,960 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855299, jitterRate=0.0875701755285263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:53:59,960 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:53:59,961 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402439941Writing region info on filesystem at 1732402439941Initializing all the Stores at 1732402439942 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439942Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439948 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402439948Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402439948Cleaning up temporary data from old regions at 1732402439957 (+9 ms)Running coprocessor post-open hooks at 1732402439960 (+3 ms)Region opened successfully at 1732402439961 (+1 ms)
2024-11-23T22:53:59,962 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402439923
2024-11-23T22:53:59,964 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:53:59,964 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:53:59,965 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,40259,1732402438754
2024-11-23T22:53:59,966 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,40259,1732402438754, state=OPEN
2024-11-23T22:54:00,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:54:00,014 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:54:00,014 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40259,1732402438754
2024-11-23T22:54:00,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:00,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:00,017 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:54:00,017 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40259,1732402438754 in 244 msec
2024-11-23T22:54:00,020 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:54:00,020 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 462 msec
2024-11-23T22:54:00,021 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:54:00,021 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:54:00,022 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:54:00,022 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,40259,1732402438754, seqNum=-1]
2024-11-23T22:54:00,023 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:54:00,024 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46863, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:54:00,029 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 938 msec
2024-11-23T22:54:00,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402440030, completionTime=-1
2024-11-23T22:54:00,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:54:00,030 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:54:00,031 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:54:00,031 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402500031
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402560031
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:46645, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,032 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,034 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.214sec
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:54:00,036 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:54:00,039 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:54:00,039 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:54:00,039 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46645,1732402438555-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:00,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6086fda4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:00,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,46645,-1 for getting cluster id
2024-11-23T22:54:00,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:54:00,084 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3894aea4-cd09-4ca0-b0cc-9b1159a1bd56'
2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3894aea4-cd09-4ca0-b0cc-9b1159a1bd56"

2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6b7368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,46645,-1]
2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:54:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:00,087 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:54:00,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322879d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:00,088 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:54:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,40259,1732402438754, seqNum=-1]
2024-11-23T22:54:00,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:54:00,092 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:54:00,094 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,46645,1732402438555
2024-11-23T22:54:00,095 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:00,097 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:54:00,098 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-23T22:54:00,098 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-23T22:54:00,098 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-23T22:54:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,46645,1732402438555
2024-11-23T22:54:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@175001ee
2024-11-23T22:54:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-23T22:54:00,101 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-23T22:54:00,102 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-23T22:54:00,102 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-23T22:54:00,102 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:54:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-23T22:54:00,105 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-23T22:54:00,105 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:00,105 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-23T22:54:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:54:00,106 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-23T22:54:00,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741835_1011 (size=395)
2024-11-23T22:54:00,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741835_1011 (size=395)
2024-11-23T22:54:00,117 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c3e48ca9044c7fb94a07a32a86885fcc, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc
2024-11-23T22:54:00,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741836_1012 (size=78)
2024-11-23T22:54:00,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741836_1012 (size=78)
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing c3e48ca9044c7fb94a07a32a86885fcc, disabling compactions & flushes
2024-11-23T22:54:00,127 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc. after waiting 0 ms
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,127 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,127 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for c3e48ca9044c7fb94a07a32a86885fcc:
Waiting for close lock at 1732402440127Disabling compacts and flushes for region at 1732402440127Disabling writes for close at 1732402440127Writing region close event to WAL at 1732402440127Closed at 1732402440127
2024-11-23T22:54:00,128 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-23T22:54:00,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732402440128"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402440128"}]},"ts":"1732402440128"}
2024-11-23T22:54:00,131 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-23T22:54:00,132 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-23T22:54:00,132 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402440132"}]},"ts":"1732402440132"}
2024-11-23T22:54:00,134 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-23T22:54:00,134 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=c3e48ca9044c7fb94a07a32a86885fcc, ASSIGN}]
2024-11-23T22:54:00,136 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=c3e48ca9044c7fb94a07a32a86885fcc, ASSIGN
2024-11-23T22:54:00,137 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=c3e48ca9044c7fb94a07a32a86885fcc, ASSIGN; state=OFFLINE, location=dd96e8dc8369,40259,1732402438754; forceNewPlan=false, retain=false
2024-11-23T22:54:00,140 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:00,287 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c3e48ca9044c7fb94a07a32a86885fcc, regionState=OPENING, regionLocation=dd96e8dc8369,40259,1732402438754
2024-11-23T22:54:00,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=c3e48ca9044c7fb94a07a32a86885fcc, ASSIGN because future has completed
2024-11-23T22:54:00,291 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e48ca9044c7fb94a07a32a86885fcc, server=dd96e8dc8369,40259,1732402438754}]
2024-11-23T22:54:00,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:00,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:00,451 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,451 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c3e48ca9044c7fb94a07a32a86885fcc, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:54:00,452 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,452 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:00,452 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,452 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,454 INFO  [StoreOpener-c3e48ca9044c7fb94a07a32a86885fcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c3e48ca9044c7fb94a07a32a86885fcc 
2024-11-23T22:54:00,456 INFO  [StoreOpener-c3e48ca9044c7fb94a07a32a86885fcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e48ca9044c7fb94a07a32a86885fcc columnFamilyName info
2024-11-23T22:54:00,456 DEBUG [StoreOpener-c3e48ca9044c7fb94a07a32a86885fcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:00,457 INFO  [StoreOpener-c3e48ca9044c7fb94a07a32a86885fcc-1 {}] regionserver.HStore(327): Store=c3e48ca9044c7fb94a07a32a86885fcc/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:54:00,457 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,458 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,459 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,459 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,459 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,461 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,464 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:54:00,465 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c3e48ca9044c7fb94a07a32a86885fcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=774020, jitterRate=-0.015783846378326416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:54:00,465 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:00,466 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c3e48ca9044c7fb94a07a32a86885fcc:
Running coprocessor pre-open hook at 1732402440452Writing region info on filesystem at 1732402440452Initializing all the Stores at 1732402440453 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402440454 (+1 ms)Cleaning up temporary data from old regions at 1732402440459 (+5 ms)Running coprocessor post-open hooks at 1732402440465 (+6 ms)Region opened successfully at 1732402440466 (+1 ms)
2024-11-23T22:54:00,468 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc., pid=6, masterSystemTime=1732402440445
2024-11-23T22:54:00,472 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,473 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c3e48ca9044c7fb94a07a32a86885fcc, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,40259,1732402438754
2024-11-23T22:54:00,472 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:00,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e48ca9044c7fb94a07a32a86885fcc, server=dd96e8dc8369,40259,1732402438754 because future has completed
2024-11-23T22:54:00,483 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-23T22:54:00,484 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c3e48ca9044c7fb94a07a32a86885fcc, server=dd96e8dc8369,40259,1732402438754 in 188 msec
2024-11-23T22:54:00,491 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-23T22:54:00,491 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=c3e48ca9044c7fb94a07a32a86885fcc, ASSIGN in 350 msec
2024-11-23T22:54:00,491 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-23T22:54:00,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402440491"}]},"ts":"1732402440491"}
2024-11-23T22:54:00,494 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-23T22:54:00,495 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-23T22:54:00,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 393 msec
2024-11-23T22:54:01,141 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:01,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:01,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:01,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:54:01,913 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-23T22:54:01,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-23T22:54:01,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-23T22:54:01,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:54:01,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-23T22:54:02,141 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:02,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:02,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:03,142 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:03,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:03,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:04,143 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:04,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:04,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:05,143 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:05,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:05,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:05,444 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:54:05,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,466 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,466 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,469 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:05,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:54:05,476 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-23T22:54:06,144 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:06,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:06,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:07,145 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:07,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:07,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:08,145 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:08,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:08,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:09,146 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:09,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:09,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:10,147 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46645 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:54:10,156 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-23T22:54:10,156 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-23T22:54:10,159 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-23T22:54:10,159 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc., hostname=dd96e8dc8369,40259,1732402438754, seqNum=2]
2024-11-23T22:54:10,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:10,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:11,147 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:11,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:11,430 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:12,148 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:12,168 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:12,170 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,170 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,170 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,171 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK], DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]) is bad.
2024-11-23T22:54:12,171 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK], DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]) is bad.
2024-11-23T22:54:12,171 WARN  [PacketResponder: BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41523] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,171 WARN  [PacketResponder: BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41523] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK], DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41523,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]) is bad.
2024-11-23T22:54:12,172 WARN  [PacketResponder: BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41523] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:34912 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34912 dst: /127.0.0.1:35405
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1881605632_22 at /127.0.0.1:34876 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34876 dst: /127.0.0.1:35405
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:34910 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34910 dst: /127.0.0.1:35405
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51662 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51662 dst: /127.0.0.1:41523
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1881605632_22 at /127.0.0.1:51626 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51626 dst: /127.0.0.1:41523
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51654 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51654 dst: /127.0.0.1:41523
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b7c275b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:12,251 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7ca0c1f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:12,251 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:12,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58716200{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:12,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32ad5d3f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:12,252 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:12,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:12,252 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:12,252 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid e9608037-2c6a-429f-8aeb-3484ce18a90b) service to localhost/127.0.0.1:41217
2024-11-23T22:54:12,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data3/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:12,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data4/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:12,253 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:12,262 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:12,265 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:12,266 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:12,266 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:12,266 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:54:12,267 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54dc8c82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:12,267 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6844f15a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:12,361 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@552dccee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-39097-hadoop-hdfs-3_4_1-tests_jar-_-any-3398601520745329059/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:12,361 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4099a6f1{HTTP/1.1, (http/1.1)}{localhost:39097}
2024-11-23T22:54:12,361 INFO  [Time-limited test {}] server.Server(415): Started @177686ms
2024-11-23T22:54:12,362 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:12,379 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,379 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,379 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:12,380 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51218 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51218 dst: /127.0.0.1:35405
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,379 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1881605632_22 at /127.0.0.1:51234 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51234 dst: /127.0.0.1:35405
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,380 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51204 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35405:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51204 dst: /127.0.0.1:35405
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:12,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@245efd98{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:12,381 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@200ef94f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:12,381 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:12,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48e9bd26{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:12,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2eb03db7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:12,383 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:12,383 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:12,383 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid b117a560-adde-472b-84be-64731583c2e3) service to localhost/127.0.0.1:41217
2024-11-23T22:54:12,383 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:12,383 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data1/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:12,383 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data2/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:12,384 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:12,393 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:12,397 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:12,398 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:12,398 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:12,398 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:54:12,398 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b350de2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:12,399 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1663a1b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:12,411 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:12,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:12,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@223db456{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-40467-hadoop-hdfs-3_4_1-tests_jar-_-any-5827255684351458741/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:12,499 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54a6c5ff{HTTP/1.1, (http/1.1)}{localhost:40467}
2024-11-23T22:54:12,499 INFO  [Time-limited test {}] server.Server(415): Started @177823ms
2024-11-23T22:54:12,500 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:12,991 WARN  [Thread-1355 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:12,994 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bb86d55a0721878 with lease ID 0x1497b22ae0d82f08: from storage DS-d8aa46c0-bbf1-4e21-b773-12b31d983533 node DatanodeRegistration(127.0.0.1:37261, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=36737, infoSecurePort=0, ipcPort=42193, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:12,995 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bb86d55a0721878 with lease ID 0x1497b22ae0d82f08: from storage DS-f8491553-8210-4e5f-b182-ac4dc48c97ae node DatanodeRegistration(127.0.0.1:37261, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=36737, infoSecurePort=0, ipcPort=42193, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:13,149 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:13,155 WARN  [Thread-1375 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:13,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6c2c770d48c471b with lease ID 0x1497b22ae0d82f09: from storage DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff node DatanodeRegistration(127.0.0.1:45203, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=33225, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:13,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6c2c770d48c471b with lease ID 0x1497b22ae0d82f09: from storage DS-6a21eede-6947-45f8-b0ef-dc3590a9a56a node DatanodeRegistration(127.0.0.1:45203, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=33225, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:13,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:13,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:13,550 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-23T22:54:13,555 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-23T22:54:13,558 ERROR [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:13,558 WARN  [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:13,559 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C40259%2C1732402438754:(num 1732402439386) roll requested
2024-11-23T22:54:13,559 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:13,565 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 newFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:13,565 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:13,565 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:13,565 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:13,565 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:13,565 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:13,566 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:13,567 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:13,567 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:13,567 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:13,568 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33225:33225),(127.0.0.1/127.0.0.1:36737:36737)]
2024-11-23T22:54:13,568 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 is not closed yet, will try archiving it next time
2024-11-23T22:54:13,568 WARN  [IPC Server handler 2 on default port 41217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-23T22:54:13,568 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 after 1ms
2024-11-23T22:54:14,150 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:14,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:14,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:14,994 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-23T22:54:15,151 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:15,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:15,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:15,571 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-23T22:54:16,152 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:16,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:16,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:17,153 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:17,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:17,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:17,569 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 after 4002ms
2024-11-23T22:54:17,575 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:17,576 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45203,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK], DatanodeInfoWithStorage[127.0.0.1:37261,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45203,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]) is bad.
2024-11-23T22:54:17,576 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:34474 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:45203:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34474 dst: /127.0.0.1:45203
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:17,576 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51502 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37261:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51502 dst: /127.0.0.1:37261
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:17,651 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@223db456{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:17,652 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54a6c5ff{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:17,652 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:17,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1663a1b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:17,653 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b350de2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:17,655 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:17,655 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid b117a560-adde-472b-84be-64731583c2e3) service to localhost/127.0.0.1:41217
2024-11-23T22:54:17,656 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data1/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:17,656 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:17,656 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:17,657 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data2/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:17,657 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:17,668 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:17,671 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:17,671 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:17,671 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:17,671 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:54:17,672 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@581007a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:17,672 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27981e4b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:17,765 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7689d41a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-37387-hadoop-hdfs-3_4_1-tests_jar-_-any-11837537588289966417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:17,766 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26044b6e{HTTP/1.1, (http/1.1)}{localhost:37387}
2024-11-23T22:54:17,766 INFO  [Time-limited test {}] server.Server(415): Started @183091ms
2024-11-23T22:54:17,767 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:17,785 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:17,785 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1930824271_22 at /127.0.0.1:51530 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37261:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51530 dst: /127.0.0.1:37261
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:17,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@552dccee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:17,790 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4099a6f1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:17,790 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:17,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6844f15a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:17,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54dc8c82{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:17,792 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:17,792 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:17,792 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:17,792 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid e9608037-2c6a-429f-8aeb-3484ce18a90b) service to localhost/127.0.0.1:41217
2024-11-23T22:54:17,793 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data3/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:17,793 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data4/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:17,793 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:17,804 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:17,807 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:17,808 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:17,808 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:17,808 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:54:17,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce4d2e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:17,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b09a4fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:17,911 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b3ddecc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/java.io.tmpdir/jetty-localhost-33633-hadoop-hdfs-3_4_1-tests_jar-_-any-16860492774718422594/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:17,912 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c48db64{HTTP/1.1, (http/1.1)}{localhost:33633}
2024-11-23T22:54:17,912 INFO  [Time-limited test {}] server.Server(415): Started @183236ms
2024-11-23T22:54:17,913 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:18,154 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:18,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:18,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:18,621 WARN  [Thread-1429 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:18,624 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e36a80dc179d76c with lease ID 0x1497b22ae0d82f0a: from storage DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff node DatanodeRegistration(127.0.0.1:37939, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=43177, infoSecurePort=0, ipcPort=36681, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:18,624 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e36a80dc179d76c with lease ID 0x1497b22ae0d82f0a: from storage DS-6a21eede-6947-45f8-b0ef-dc3590a9a56a node DatanodeRegistration(127.0.0.1:37939, datanodeUuid=b117a560-adde-472b-84be-64731583c2e3, infoPort=43177, infoSecurePort=0, ipcPort=36681, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:18,782 WARN  [Thread-1449 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:18,784 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc518831a96b4b02 with lease ID 0x1497b22ae0d82f0b: from storage DS-d8aa46c0-bbf1-4e21-b773-12b31d983533 node DatanodeRegistration(127.0.0.1:39897, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=37179, infoSecurePort=0, ipcPort=33781, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:18,784 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc518831a96b4b02 with lease ID 0x1497b22ae0d82f0b: from storage DS-f8491553-8210-4e5f-b182-ac4dc48c97ae node DatanodeRegistration(127.0.0.1:39897, datanodeUuid=e9608037-2c6a-429f-8aeb-3484ce18a90b, infoPort=37179, infoSecurePort=0, ipcPort=33781, storageInfo=lv=-57;cid=testClusterID;nsid=502175931;c=1732402436130), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:18,932 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-23T22:54:18,934 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-23T22:54:18,936 ERROR [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37261,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:18,936 WARN  [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37261,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:18,936 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C40259%2C1732402438754:(num 1732402453559) roll requested
2024-11-23T22:54:18,936 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:18,942 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 newFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:18,943 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:18,943 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:18,943 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:18,943 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:18,943 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:18,943 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:18,944 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37261,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:18,944 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37261,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:18,944 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:18,944 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43177:43177),(127.0.0.1/127.0.0.1:37179:37179)]
2024-11-23T22:54:18,944 WARN  [IPC Server handler 3 on default port 41217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-23T22:54:18,944 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 is not closed yet, will try archiving it next time
2024-11-23T22:54:18,945 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 after 1ms
2024-11-23T22:54:19,154 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:19,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:19,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:20,155 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:20,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:20,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:20,946 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:20,951 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936 newFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:20,951 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:20,952 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:20,952 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:20,952 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:20,952 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:20,952 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:20,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37179:37179),(127.0.0.1/127.0.0.1:43177:43177)]
2024-11-23T22:54:20,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 is not closed yet, will try archiving it next time
2024-11-23T22:54:20,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936 is not closed yet, will try archiving it next time
2024-11-23T22:54:20,953 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:20,953 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:20,954 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386 after 1ms
2024-11-23T22:54:20,954 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:20,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741838_1019 (size=1264)
2024-11-23T22:54:20,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741838_1019 (size=1264)
2024-11-23T22:54:20,955 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 is not closed yet, will try archiving it next time
2024-11-23T22:54:20,963 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732402440466/Put/vlen=218/seqid=0]
2024-11-23T22:54:20,963 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732402450166/Put/vlen=1045/seqid=0]
2024-11-23T22:54:20,963 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402439386
2024-11-23T22:54:20,963 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:20,963 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:20,964 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 after 1ms
2024-11-23T22:54:20,964 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:20,968 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732402453557/Put/vlen=1045/seqid=0]
2024-11-23T22:54:20,968 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732402455573/Put/vlen=1045/seqid=0]
2024-11-23T22:54:20,968 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559
2024-11-23T22:54:20,968 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:20,968 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:20,969 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936 after 1ms
2024-11-23T22:54:20,969 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402458936
2024-11-23T22:54:20,972 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732402458935/Put/vlen=1045/seqid=0]
2024-11-23T22:54:20,972 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:20,972 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:20,973 WARN  [IPC Server handler 3 on default port 41217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-23T22:54:20,973 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 after 1ms
2024-11-23T22:54:21,156 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:21,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:21,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:21,787 WARN  [ResponseProcessor for block BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:21,787 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1881605632_22 at /127.0.0.1:49312 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39897:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49312 dst: /127.0.0.1:39897
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39897 remote=/127.0.0.1:49312]. Total timeout mills is 60000, 59164 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:21,788 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 block BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39897,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK], DatanodeInfoWithStorage[127.0.0.1:37939,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39897,DS-d8aa46c0-bbf1-4e21-b773-12b31d983533,DISK]) is bad.
2024-11-23T22:54:21,787 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1881605632_22 at /127.0.0.1:33654 [Receiving block BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37939:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33654 dst: /127.0.0.1:37939
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:21,789 WARN  [DataStreamer for file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 block BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:21,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741839_1022 (size=85)
2024-11-23T22:54:21,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741839_1022 (size=85)
2024-11-23T22:54:22,156 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:22,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:22,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:22,947 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402453559 after 4003ms
2024-11-23T22:54:23,157 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:23,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:23,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:24,158 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:24,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:24,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:24,625 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-23T22:54:24,974 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 after 4002ms
2024-11-23T22:54:24,974 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:24,980 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:24,980 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-23T22:54:24,981 ERROR [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:24,981 WARN  [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:24,981 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C40259%2C1732402438754.meta:.meta(num 1732402439932) roll requested
2024-11-23T22:54:24,982 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.meta.1732402464981.meta
2024-11-23T22:54:24,988 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:24,988 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:24,988 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:24,988 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:24,988 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:24,989 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402464981.meta
2024-11-23T22:54:24,989 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:24,989 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:24,989 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta
2024-11-23T22:54:24,990 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43177:43177),(127.0.0.1/127.0.0.1:37179:37179)]
2024-11-23T22:54:24,990 WARN  [IPC Server handler 2 on default port 41217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-11-23T22:54:24,990 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta is not closed yet, will try archiving it next time
2024-11-23T22:54:24,990 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta after 1ms
2024-11-23T22:54:25,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/info/104dc9d60c9746ae9c43a0edb4efe444 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc./info:regioninfo/1732402440472/Put/seqid=0
2024-11-23T22:54:25,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741841_1025 (size=7125)
2024-11-23T22:54:25,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741841_1025 (size=7125)
2024-11-23T22:54:25,011 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/info/104dc9d60c9746ae9c43a0edb4efe444
2024-11-23T22:54:25,031 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/ns/ad2d0a683bd847fabba3d1b127b48ffd is 43, key is default/ns:d/1732402440024/Put/seqid=0
2024-11-23T22:54:25,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741842_1026 (size=5153)
2024-11-23T22:54:25,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741842_1026 (size=5153)
2024-11-23T22:54:25,036 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/ns/ad2d0a683bd847fabba3d1b127b48ffd
2024-11-23T22:54:25,053 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/table/aae73b9da8634f5d8d6ac98229668827 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732402440491/Put/seqid=0
2024-11-23T22:54:25,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741843_1027 (size=5438)
2024-11-23T22:54:25,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741843_1027 (size=5438)
2024-11-23T22:54:25,059 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/table/aae73b9da8634f5d8d6ac98229668827
2024-11-23T22:54:25,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/info/104dc9d60c9746ae9c43a0edb4efe444 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/info/104dc9d60c9746ae9c43a0edb4efe444
2024-11-23T22:54:25,071 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/info/104dc9d60c9746ae9c43a0edb4efe444, entries=10, sequenceid=11, filesize=7.0 K
2024-11-23T22:54:25,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/ns/ad2d0a683bd847fabba3d1b127b48ffd as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/ns/ad2d0a683bd847fabba3d1b127b48ffd
2024-11-23T22:54:25,077 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/ns/ad2d0a683bd847fabba3d1b127b48ffd, entries=2, sequenceid=11, filesize=5.0 K
2024-11-23T22:54:25,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/.tmp/table/aae73b9da8634f5d8d6ac98229668827 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/table/aae73b9da8634f5d8d6ac98229668827
2024-11-23T22:54:25,086 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/table/aae73b9da8634f5d8d6ac98229668827, entries=2, sequenceid=11, filesize=5.3 K
2024-11-23T22:54:25,087 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=11, compaction requested=false
2024-11-23T22:54:25,088 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-23T22:54:25,088 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing c3e48ca9044c7fb94a07a32a86885fcc 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-23T22:54:25,088 ERROR [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:25,088 WARN  [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc-prefix:dd96e8dc8369,40259,1732402438754 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:25,089 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C40259%2C1732402438754:(num 1732402460946) roll requested
2024-11-23T22:54:25,089 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40259%2C1732402438754.1732402465089
2024-11-23T22:54:25,094 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 newFile=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402465089
2024-11-23T22:54:25,094 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,094 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,094 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,095 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,095 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,095 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402465089
2024-11-23T22:54:25,095 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:25,095 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37179:37179),(127.0.0.1/127.0.0.1:43177:43177)]
2024-11-23T22:54:25,095 DEBUG [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 is not closed yet, will try archiving it next time
2024-11-23T22:54:25,095 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1031033842-172.17.0.2-1732402436130:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:25,095 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:25,096 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 after 1ms
2024-11-23T22:54:25,096 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.1732402460946 to hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs/dd96e8dc8369%2C40259%2C1732402438754.1732402460946
2024-11-23T22:54:25,110 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/.tmp/info/0a6af0f967974749bf39816a4b053c63 is 1080, key is row1002/info:/1732402450166/Put/seqid=0
2024-11-23T22:54:25,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741845_1029 (size=9270)
2024-11-23T22:54:25,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741845_1029 (size=9270)
2024-11-23T22:54:25,115 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/.tmp/info/0a6af0f967974749bf39816a4b053c63
2024-11-23T22:54:25,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/.tmp/info/0a6af0f967974749bf39816a4b053c63 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/info/0a6af0f967974749bf39816a4b053c63
2024-11-23T22:54:25,126 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/info/0a6af0f967974749bf39816a4b053c63, entries=4, sequenceid=8, filesize=9.1 K
2024-11-23T22:54:25,128 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for c3e48ca9044c7fb94a07a32a86885fcc in 39ms, sequenceid=8, compaction requested=false
2024-11-23T22:54:25,128 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c3e48ca9044c7fb94a07a32a86885fcc:

2024-11-23T22:54:25,133 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:54:25,133 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:54:25,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:54:25,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:25,134 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:54:25,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:25,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:54:25,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=975957115, stopped=false
2024-11-23T22:54:25,134 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,46645,1732402438555
2024-11-23T22:54:25,158 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:25,189 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:54:25,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:54:25,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:25,189 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:25,189 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:54:25,190 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:54:25,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:54:25,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:25,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:54:25,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:54:25,190 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,40259,1732402438754' *****
2024-11-23T22:54:25,190 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:54:25,190 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:54:25,190 INFO  [RS:0;dd96e8dc8369:40259 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:54:25,190 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(3091): Received CLOSE for c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,40259,1732402438754
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:40259.
2024-11-23T22:54:25,191 DEBUG [RS:0;dd96e8dc8369:40259 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:54:25,191 DEBUG [RS:0;dd96e8dc8369:40259 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:25,191 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c3e48ca9044c7fb94a07a32a86885fcc, disabling compactions & flushes
2024-11-23T22:54:25,191 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:54:25,191 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:54:25,191 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc. after waiting 0 ms
2024-11-23T22:54:25,191 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:25,191 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-23T22:54:25,191 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c3e48ca9044c7fb94a07a32a86885fcc=TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.}
2024-11-23T22:54:25,191 DEBUG [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c3e48ca9044c7fb94a07a32a86885fcc
2024-11-23T22:54:25,191 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:54:25,192 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:54:25,192 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:54:25,192 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:54:25,192 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:54:25,195 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-23T22:54:25,195 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/data/default/TestLogRolling-testLogRollOnPipelineRestart/c3e48ca9044c7fb94a07a32a86885fcc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-23T22:54:25,196 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:54:25,196 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:25,196 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:54:25,196 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402465191Running coprocessor pre-close hooks at 1732402465191Disabling compacts and flushes for region at 1732402465191Disabling writes for close at 1732402465192 (+1 ms)Writing region close event to WAL at 1732402465192Running coprocessor post-close hooks at 1732402465195 (+3 ms)Closed at 1732402465196 (+1 ms)
2024-11-23T22:54:25,196 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c3e48ca9044c7fb94a07a32a86885fcc:
Waiting for close lock at 1732402465191Running coprocessor pre-close hooks at 1732402465191Disabling compacts and flushes for region at 1732402465191Disabling writes for close at 1732402465191Writing region close event to WAL at 1732402465192 (+1 ms)Running coprocessor post-close hooks at 1732402465196 (+4 ms)Closed at 1732402465196
2024-11-23T22:54:25,196 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:54:25,196 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732402440102.c3e48ca9044c7fb94a07a32a86885fcc.
2024-11-23T22:54:25,253 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:54:25,256 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-23T22:54:25,256 INFO  [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-23T22:54:25,392 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,40259,1732402438754; all regions closed.
2024-11-23T22:54:25,392 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,392 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,392 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,393 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,393 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:25,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741840_1023 (size=825)
2024-11-23T22:54:25,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741840_1023 (size=825)
2024-11-23T22:54:25,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:25,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:26,159 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:26,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:26,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:27,160 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:27,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431 after 68062ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:27,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:27,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:27,786 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-23T22:54:28,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:28,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:28,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:28,531 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:54:28,991 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta after 4002ms
2024-11-23T22:54:28,991 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/WALs/dd96e8dc8369,40259,1732402438754/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta to hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs/dd96e8dc8369%2C40259%2C1732402438754.meta.1732402439932.meta
2024-11-23T22:54:28,994 DEBUG [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs
2024-11-23T22:54:28,995 INFO  [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C40259%2C1732402438754.meta:.meta(num 1732402464981)
2024-11-23T22:54:28,995 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:28,995 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:28,995 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:28,995 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:28,995 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:28,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741844_1028 (size=1162)
2024-11-23T22:54:28,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741844_1028 (size=1162)
2024-11-23T22:54:29,003 DEBUG [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs
2024-11-23T22:54:29,003 INFO  [RS:0;dd96e8dc8369:40259 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C40259%2C1732402438754:(num 1732402465089)
2024-11-23T22:54:29,003 DEBUG [RS:0;dd96e8dc8369:40259 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:29,003 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:54:29,003 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:54:29,003 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:54:29,004 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:54:29,004 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:54:29,004 INFO  [RS:0;dd96e8dc8369:40259 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40259
2024-11-23T22:54:29,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:54:29,031 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,40259,1732402438754
2024-11-23T22:54:29,031 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:54:29,041 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,40259,1732402438754]
2024-11-23T22:54:29,052 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,40259,1732402438754 already deleted, retry=false
2024-11-23T22:54:29,052 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,40259,1732402438754 expired; onlineServers=0
2024-11-23T22:54:29,052 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,46645,1732402438555' *****
2024-11-23T22:54:29,052 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:54:29,052 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:54:29,052 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:54:29,052 DEBUG [M:0;dd96e8dc8369:46645 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:54:29,052 DEBUG [M:0;dd96e8dc8369:46645 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:54:29,052 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:54:29,053 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402439103 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402439103,5,FailOnTimeoutGroup]
2024-11-23T22:54:29,053 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402439103 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402439103,5,FailOnTimeoutGroup]
2024-11-23T22:54:29,053 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:54:29,053 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:54:29,053 DEBUG [M:0;dd96e8dc8369:46645 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:54:29,053 INFO  [M:0;dd96e8dc8369:46645 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:54:29,053 INFO  [M:0;dd96e8dc8369:46645 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:54:29,053 INFO  [M:0;dd96e8dc8369:46645 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:54:29,053 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:54:29,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:54:29,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:29,063 DEBUG [M:0;dd96e8dc8369:46645 {}] zookeeper.ZKUtil(347): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:54:29,063 WARN  [M:0;dd96e8dc8369:46645 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:54:29,063 INFO  [M:0;dd96e8dc8369:46645 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/.lastflushedseqids
2024-11-23T22:54:29,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741846_1030 (size=111)
2024-11-23T22:54:29,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741846_1030 (size=111)
2024-11-23T22:54:29,069 INFO  [M:0;dd96e8dc8369:46645 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:54:29,069 INFO  [M:0;dd96e8dc8369:46645 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:54:29,069 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:54:29,069 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:29,069 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:29,069 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:54:29,069 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:29,070 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.15 KB heapSize=29.12 KB
2024-11-23T22:54:29,070 ERROR [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData-prefix:dd96e8dc8369,46645,1732402438555 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:29,070 WARN  [FSHLog-0-hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData-prefix:dd96e8dc8369,46645,1732402438555 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:29,070 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog dd96e8dc8369%2C46645%2C1732402438555:(num 1732402438879) roll requested
2024-11-23T22:54:29,071 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46645%2C1732402438555.1732402469070
2024-11-23T22:54:29,077 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,077 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,077 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,077 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,077 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,077 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 with entries=53, filesize=26.60 KB; new WAL /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402469070
2024-11-23T22:54:29,078 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:29,078 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35405,DS-55b1ef3a-ccc9-41d3-b464-b4802e2a7fff,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-23T22:54:29,078 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879
2024-11-23T22:54:29,078 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43177:43177),(127.0.0.1/127.0.0.1:37179:37179)]
2024-11-23T22:54:29,078 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 is not closed yet, will try archiving it next time
2024-11-23T22:54:29,079 WARN  [IPC Server handler 3 on default port 41217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-23T22:54:29,079 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 after 1ms
2024-11-23T22:54:29,097 DEBUG [M:0;dd96e8dc8369:46645 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58c30594e55741f98779b7ebf3641ef4 is 82, key is hbase:meta,,1/info:regioninfo/1732402439964/Put/seqid=0
2024-11-23T22:54:29,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741848_1033 (size=5672)
2024-11-23T22:54:29,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741848_1033 (size=5672)
2024-11-23T22:54:29,109 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58c30594e55741f98779b7ebf3641ef4
2024-11-23T22:54:29,135 DEBUG [M:0;dd96e8dc8369:46645 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf20a1b7058142aeb3dfe718c3ca29c5 is 776, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402440497/Put/seqid=0
2024-11-23T22:54:29,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741849_1034 (size=6116)
2024-11-23T22:54:29,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741849_1034 (size=6116)
2024-11-23T22:54:29,140 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf20a1b7058142aeb3dfe718c3ca29c5
2024-11-23T22:54:29,142 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:54:29,142 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40259-0x10169e75d320001, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:54:29,142 INFO  [RS:0;dd96e8dc8369:40259 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:54:29,142 INFO  [RS:0;dd96e8dc8369:40259 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,40259,1732402438754; zookeeper connection closed.
2024-11-23T22:54:29,142 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2610d21e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2610d21e
2024-11-23T22:54:29,142 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:54:29,157 DEBUG [M:0;dd96e8dc8369:46645 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a9ee9c6fd084425a6a81f9ab787aebf is 69, key is dd96e8dc8369,40259,1732402438754/rs:state/1732402439228/Put/seqid=0
2024-11-23T22:54:29,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741850_1035 (size=5156)
2024-11-23T22:54:29,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741850_1035 (size=5156)
2024-11-23T22:54:29,162 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:29,163 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a9ee9c6fd084425a6a81f9ab787aebf
2024-11-23T22:54:29,181 DEBUG [M:0;dd96e8dc8369:46645 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7c48fff39a49b0907c51073e83c163 is 52, key is load_balancer_on/state:d/1732402440096/Put/seqid=0
2024-11-23T22:54:29,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741851_1036 (size=5056)
2024-11-23T22:54:29,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741851_1036 (size=5056)
2024-11-23T22:54:29,187 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7c48fff39a49b0907c51073e83c163
2024-11-23T22:54:29,192 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/58c30594e55741f98779b7ebf3641ef4 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58c30594e55741f98779b7ebf3641ef4
2024-11-23T22:54:29,197 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/58c30594e55741f98779b7ebf3641ef4, entries=8, sequenceid=56, filesize=5.5 K
2024-11-23T22:54:29,198 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf20a1b7058142aeb3dfe718c3ca29c5 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bf20a1b7058142aeb3dfe718c3ca29c5
2024-11-23T22:54:29,203 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bf20a1b7058142aeb3dfe718c3ca29c5, entries=6, sequenceid=56, filesize=6.0 K
2024-11-23T22:54:29,204 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a9ee9c6fd084425a6a81f9ab787aebf as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a9ee9c6fd084425a6a81f9ab787aebf
2024-11-23T22:54:29,210 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a9ee9c6fd084425a6a81f9ab787aebf, entries=1, sequenceid=56, filesize=5.0 K
2024-11-23T22:54:29,211 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7c48fff39a49b0907c51073e83c163 as hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec7c48fff39a49b0907c51073e83c163
2024-11-23T22:54:29,216 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec7c48fff39a49b0907c51073e83c163, entries=1, sequenceid=56, filesize=4.9 K
2024-11-23T22:54:29,217 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.15 KB/23702, heapSize ~29.05 KB/29752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=56, compaction requested=false
2024-11-23T22:54:29,219 INFO  [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:29,219 DEBUG [M:0;dd96e8dc8369:46645 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402469069Disabling compacts and flushes for region at 1732402469069Disabling writes for close at 1732402469069Obtaining lock to block concurrent updates at 1732402469070 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402469070Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23702, getHeapSize=29752, getOffHeapSize=0, getCellsCount=67 at 1732402469070Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402469079 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402469079Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402469097 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402469097Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402469118 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402469134 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402469134Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402469144 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402469157 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402469157Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402469167 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402469180 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402469180Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63994c44: reopening flushed file at 1732402469191 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324ae58d: reopening flushed file at 1732402469197 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f957daa: reopening flushed file at 1732402469204 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@582d3902: reopening flushed file at 1732402469210 (+6 ms)Finished flush of dataSize ~23.15 KB/23702, heapSize ~29.05 KB/29752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=56, compaction requested=false at 1732402469217 (+7 ms)Writing region close event to WAL at 1732402469219 (+2 ms)Closed at 1732402469219
2024-11-23T22:54:29,219 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,219 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,219 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,219 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,219 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:54:29,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39897 is added to blk_1073741847_1031 (size=757)
2024-11-23T22:54:29,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37939 is added to blk_1073741847_1031 (size=757)
2024-11-23T22:54:29,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:29,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:30,163 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:30,197 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,198 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,220 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,220 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,220 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,220 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,221 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,221 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,226 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:30,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:30,733 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:54:30,734 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,762 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,763 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,763 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:30,786 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-23T22:54:31,164 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:31,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:31,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:54:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:54:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-23T22:54:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-23T22:54:32,165 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:32,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:32,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:33,081 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 after 4003ms
2024-11-23T22:54:33,082 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/WALs/dd96e8dc8369,46645,1732402438555/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 to hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/oldWALs/dd96e8dc8369%2C46645%2C1732402438555.1732402438879
2024-11-23T22:54:33,090 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/MasterData/oldWALs/dd96e8dc8369%2C46645%2C1732402438555.1732402438879 to hdfs://localhost:41217/user/jenkins/test-data/89469fc8-d64f-931a-f113-eeb7e5d20bfc/oldWALs/dd96e8dc8369%2C46645%2C1732402438555.1732402438879$masterlocalwal$
2024-11-23T22:54:33,091 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:54:33,091 INFO  [M:0;dd96e8dc8369:46645 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:54:33,091 INFO  [M:0;dd96e8dc8369:46645 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46645
2024-11-23T22:54:33,091 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:54:33,165 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:54:33,242 INFO  [M:0;dd96e8dc8369:46645 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:54:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46645-0x10169e75d320000, quorum=127.0.0.1:63019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:54:33,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b3ddecc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:33,245 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c48db64{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:33,245 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:33,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b09a4fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:33,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce4d2e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:33,246 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:33,246 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:33,246 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid e9608037-2c6a-429f-8aeb-3484ce18a90b) service to localhost/127.0.0.1:41217
2024-11-23T22:54:33,247 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:33,247 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data3/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:33,247 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data4/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:33,247 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:33,252 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7689d41a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:33,253 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26044b6e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:33,253 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:33,253 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27981e4b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:33,253 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@581007a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:33,254 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:54:33,254 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:54:33,254 WARN  [BP-1031033842-172.17.0.2-1732402436130 heartbeating to localhost/127.0.0.1:41217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1031033842-172.17.0.2-1732402436130 (Datanode Uuid b117a560-adde-472b-84be-64731583c2e3) service to localhost/127.0.0.1:41217
2024-11-23T22:54:33,254 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:54:33,255 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data1/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:33,255 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/cluster_6122048b-2e5c-7e93-b11b-dd6f92558d09/data/data2/current/BP-1031033842-172.17.0.2-1732402436130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:54:33,255 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:54:33,261 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@318626e1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:54:33,261 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11ec3e38{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:54:33,262 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:54:33,262 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73f89830{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:54:33,262 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2720e386{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir/,STOPPED}
2024-11-23T22:54:33,268 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:54:33,285 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:54:33,292 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=183 (was 158)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:41217
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:41217 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:41217 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41217
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:41217 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 438) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=106 (was 151), ProcessCount=11 (was 11), AvailableMemoryMB=7000 (was 6801) - AvailableMemoryMB LEAK? -
2024-11-23T22:54:33,299 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=183, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=106, ProcessCount=11, AvailableMemoryMB=7000
2024-11-23T22:54:33,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:54:33,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.log.dir so I do NOT create it in target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661
2024-11-23T22:54:33,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eb5e51dd-6fc6-e2e0-5620-fbd234cd20cf/hadoop.tmp.dir so I do NOT create it in target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661
2024-11-23T22:54:33,299 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc, deleteOnExit=true
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/test.cache.data in system properties and HBase conf
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:54:33,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:54:33,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:54:33,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:54:33,302 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:54:33,314 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:54:33,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:33,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:33,675 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:33,679 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:33,680 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:33,680 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:33,680 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:54:33,680 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:33,681 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38b36302{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:33,681 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e264d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:33,771 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58d2b6e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/java.io.tmpdir/jetty-localhost-44561-hadoop-hdfs-3_4_1-tests_jar-_-any-198876987836342438/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:54:33,772 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14642123{HTTP/1.1, (http/1.1)}{localhost:44561}
2024-11-23T22:54:33,772 INFO  [Time-limited test {}] server.Server(415): Started @199097ms
2024-11-23T22:54:33,782 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:54:34,067 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:34,070 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:34,071 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:34,071 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:34,071 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:54:34,071 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f81b4d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:34,072 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@188d3049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:34,166 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:34,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cda66bc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/java.io.tmpdir/jetty-localhost-35857-hadoop-hdfs-3_4_1-tests_jar-_-any-17469308315892295534/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:34,171 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e84571d{HTTP/1.1, (http/1.1)}{localhost:35857}
2024-11-23T22:54:34,171 INFO  [Time-limited test {}] server.Server(415): Started @199496ms
2024-11-23T22:54:34,172 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:34,203 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:54:34,206 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:54:34,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:54:34,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:54:34,206 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:54:34,207 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@432bc536{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:54:34,207 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@718e5030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:54:34,301 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cb8e2e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/java.io.tmpdir/jetty-localhost-43055-hadoop-hdfs-3_4_1-tests_jar-_-any-3743628864301825995/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:54:34,302 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33803ea9{HTTP/1.1, (http/1.1)}{localhost:43055}
2024-11-23T22:54:34,302 INFO  [Time-limited test {}] server.Server(415): Started @199626ms
2024-11-23T22:54:34,303 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:54:34,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:34,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:35,167 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:35,216 WARN  [Thread-1670 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data2/current/BP-2045060364-172.17.0.2-1732402473324/current, will proceed with Du for space computation calculation, 
2024-11-23T22:54:35,216 WARN  [Thread-1669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data1/current/BP-2045060364-172.17.0.2-1732402473324/current, will proceed with Du for space computation calculation, 
2024-11-23T22:54:35,232 WARN  [Thread-1633 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:35,235 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x422e815ff72e51d with lease ID 0x6a07fd1f81e8711b: Processing first storage report for DS-c18c2d92-3107-4b7b-9dde-e9387adfd078 from datanode DatanodeRegistration(127.0.0.1:37707, datanodeUuid=3a61fb35-f0c7-47cf-b0c4-7222b1f197c6, infoPort=34649, infoSecurePort=0, ipcPort=46651, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324)
2024-11-23T22:54:35,235 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x422e815ff72e51d with lease ID 0x6a07fd1f81e8711b: from storage DS-c18c2d92-3107-4b7b-9dde-e9387adfd078 node DatanodeRegistration(127.0.0.1:37707, datanodeUuid=3a61fb35-f0c7-47cf-b0c4-7222b1f197c6, infoPort=34649, infoSecurePort=0, ipcPort=46651, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:35,235 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x422e815ff72e51d with lease ID 0x6a07fd1f81e8711b: Processing first storage report for DS-718b97f8-cfb4-4f74-880e-6ed567aa978b from datanode DatanodeRegistration(127.0.0.1:37707, datanodeUuid=3a61fb35-f0c7-47cf-b0c4-7222b1f197c6, infoPort=34649, infoSecurePort=0, ipcPort=46651, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324)
2024-11-23T22:54:35,235 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x422e815ff72e51d with lease ID 0x6a07fd1f81e8711b: from storage DS-718b97f8-cfb4-4f74-880e-6ed567aa978b node DatanodeRegistration(127.0.0.1:37707, datanodeUuid=3a61fb35-f0c7-47cf-b0c4-7222b1f197c6, infoPort=34649, infoSecurePort=0, ipcPort=46651, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:35,270 WARN  [Thread-1680 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data3/current/BP-2045060364-172.17.0.2-1732402473324/current, will proceed with Du for space computation calculation, 
2024-11-23T22:54:35,270 WARN  [Thread-1681 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data4/current/BP-2045060364-172.17.0.2-1732402473324/current, will proceed with Du for space computation calculation, 
2024-11-23T22:54:35,290 WARN  [Thread-1656 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:54:35,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x608c99eeda2b4945 with lease ID 0x6a07fd1f81e8711c: Processing first storage report for DS-10988b08-5754-4f30-972e-3db2c5f7e48f from datanode DatanodeRegistration(127.0.0.1:36151, datanodeUuid=f12b4171-7ce1-4a14-8a97-7463c2628d64, infoPort=39777, infoSecurePort=0, ipcPort=38447, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324)
2024-11-23T22:54:35,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x608c99eeda2b4945 with lease ID 0x6a07fd1f81e8711c: from storage DS-10988b08-5754-4f30-972e-3db2c5f7e48f node DatanodeRegistration(127.0.0.1:36151, datanodeUuid=f12b4171-7ce1-4a14-8a97-7463c2628d64, infoPort=39777, infoSecurePort=0, ipcPort=38447, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:35,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x608c99eeda2b4945 with lease ID 0x6a07fd1f81e8711c: Processing first storage report for DS-f785c013-879f-4042-858b-3a73231386e6 from datanode DatanodeRegistration(127.0.0.1:36151, datanodeUuid=f12b4171-7ce1-4a14-8a97-7463c2628d64, infoPort=39777, infoSecurePort=0, ipcPort=38447, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324)
2024-11-23T22:54:35,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x608c99eeda2b4945 with lease ID 0x6a07fd1f81e8711c: from storage DS-f785c013-879f-4042-858b-3a73231386e6 node DatanodeRegistration(127.0.0.1:36151, datanodeUuid=f12b4171-7ce1-4a14-8a97-7463c2628d64, infoPort=39777, infoSecurePort=0, ipcPort=38447, storageInfo=lv=-57;cid=testClusterID;nsid=1959778727;c=1732402473324), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:54:35,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661
2024-11-23T22:54:35,362 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/zookeeper_0, clientPort=59470, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:54:35,364 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59470
2024-11-23T22:54:35,365 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,367 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:54:35,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:54:35,381 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e with version=8
2024-11-23T22:54:35,381 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:54:35,383 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:54:35,383 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,383 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:54:35,384 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39775
2024-11-23T22:54:35,386 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39775 connecting to ZooKeeper ensemble=127.0.0.1:59470
2024-11-23T22:54:35,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:35,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397750x0, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:54:35,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39775-0x10169e7ed0c0000 connected
2024-11-23T22:54:35,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:35,526 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,527 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:54:35,529 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e, hbase.cluster.distributed=false
2024-11-23T22:54:35,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:54:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39775
2024-11-23T22:54:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39775
2024-11-23T22:54:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39775
2024-11-23T22:54:35,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39775
2024-11-23T22:54:35,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39775
2024-11-23T22:54:35,550 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:54:35,550 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,550 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,550 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:54:35,550 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:54:35,551 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:54:35,551 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:54:35,551 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:54:35,551 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40613
2024-11-23T22:54:35,553 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40613 connecting to ZooKeeper ensemble=127.0.0.1:59470
2024-11-23T22:54:35,553 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,555 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406130x0, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:54:35,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406130x0, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:54:35,568 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40613-0x10169e7ed0c0001 connected
2024-11-23T22:54:35,568 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:54:35,569 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:54:35,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:54:35,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:54:35,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40613
2024-11-23T22:54:35,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40613
2024-11-23T22:54:35,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40613
2024-11-23T22:54:35,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40613
2024-11-23T22:54:35,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40613
2024-11-23T22:54:35,584 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:39775
2024-11-23T22:54:35,584 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:54:35,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:54:35,595 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:54:35,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,605 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:54:35,606 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,39775,1732402475383 from backup master directory
2024-11-23T22:54:35,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:54:35,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:54:35,615 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:54:35,615 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,622 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/hbase.id] with ID: df673e81-8e0a-4e6a-8504-9551941f2b6c
2024-11-23T22:54:35,622 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/.tmp/hbase.id
2024-11-23T22:54:35,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:54:35,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:54:35,632 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/.tmp/hbase.id]:[hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/hbase.id]
2024-11-23T22:54:35,646 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:35,646 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:54:35,648 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-23T22:54:35,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:54:35,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:54:35,669 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:54:35,670 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:54:35,671 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:54:35,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:54:35,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:54:35,679 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store
2024-11-23T22:54:35,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:54:35,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:54:35,686 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:35,686 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:54:35,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402475686Disabling compacts and flushes for region at 1732402475686Disabling writes for close at 1732402475686Writing region close event to WAL at 1732402475686Closed at 1732402475686
2024-11-23T22:54:35,687 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/.initializing
2024-11-23T22:54:35,687 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/WALs/dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,690 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C39775%2C1732402475383, suffix=, logDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/WALs/dd96e8dc8369,39775,1732402475383, archiveDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/oldWALs, maxLogs=10
2024-11-23T22:54:35,691 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C39775%2C1732402475383.1732402475690
2024-11-23T22:54:35,696 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/WALs/dd96e8dc8369,39775,1732402475383/dd96e8dc8369%2C39775%2C1732402475383.1732402475690
2024-11-23T22:54:35,700 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34649:34649),(127.0.0.1/127.0.0.1:39777:39777)]
2024-11-23T22:54:35,701 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:54:35,701 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:35,701 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,701 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,703 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:54:35,704 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:54:35,704 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,705 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:35,705 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:54:35,706 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:54:35,706 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,707 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:54:35,707 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:54:35,708 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:54:35,708 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,709 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:54:35,709 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:54:35,710 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:54:35,710 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,710 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:54:35,711 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,711 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,712 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,714 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,714 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,715 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:54:35,716 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:54:35,719 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:54:35,719 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735229, jitterRate=-0.06510798633098602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:54:35,720 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402475702Initializing all the Stores at 1732402475702Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402475702Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402475703 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402475703Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402475703Cleaning up temporary data from old regions at 1732402475714 (+11 ms)Region opened successfully at 1732402475720 (+6 ms)
2024-11-23T22:54:35,721 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:54:35,725 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d34748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:54:35,726 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:54:35,726 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:54:35,726 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:54:35,727 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:54:35,727 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:54:35,728 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:54:35,728 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:54:35,730 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:54:35,731 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:54:35,741 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:54:35,742 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:54:35,743 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:54:35,752 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:54:35,753 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:54:35,755 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:54:35,768 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:54:35,769 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:54:35,778 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:54:35,784 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:54:35,794 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:54:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:54:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:54:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,805 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,39775,1732402475383, sessionid=0x10169e7ed0c0000, setting cluster-up flag (Was=false)
2024-11-23T22:54:35,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,857 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:54:35,859 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:35,915 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:54:35,916 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:35,917 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:54:35,919 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:54:35,919 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:54:35,919 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:54:35,919 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,39775,1732402475383
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:54:35,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402505922
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:54:35,922 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:35,923 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:54:35,923 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:54:35,923 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:54:35,923 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:54:35,923 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:54:35,923 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:54:35,923 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:54:35,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402475923,5,FailOnTimeoutGroup]
2024-11-23T22:54:35,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402475924,5,FailOnTimeoutGroup]
2024-11-23T22:54:35,924 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:35,924 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:54:35,924 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:35,924 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:35,924 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,924 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:54:35,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:54:35,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:54:35,933 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:54:35,933 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e
2024-11-23T22:54:35,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:54:35,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:54:35,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:35,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:54:35,945 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:54:35,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:35,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:54:35,946 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:54:35,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:35,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:54:35,948 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:54:35,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:35,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:54:35,950 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:54:35,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:35,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:35,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:54:35,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740
2024-11-23T22:54:35,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740
2024-11-23T22:54:35,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:54:35,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:54:35,954 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:54:35,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:54:35,957 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:54:35,957 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873694, jitterRate=0.11096061766147614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402475942Initializing all the Stores at 1732402475943 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402475943Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402475943Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402475943Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402475943Cleaning up temporary data from old regions at 1732402475953 (+10 ms)Region opened successfully at 1732402475958 (+5 ms)
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:54:35,958 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:54:35,958 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:54:35,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402475958Disabling compacts and flushes for region at 1732402475958Disabling writes for close at 1732402475958Writing region close event to WAL at 1732402475958Closed at 1732402475958
2024-11-23T22:54:35,959 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:54:35,959 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:54:35,960 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:54:35,961 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:54:35,962 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:54:35,974 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(746): ClusterId : df673e81-8e0a-4e6a-8504-9551941f2b6c
2024-11-23T22:54:35,974 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:54:35,985 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:54:35,985 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:54:35,995 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:54:35,995 DEBUG [RS:0;dd96e8dc8369:40613 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fd9a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:54:36,008 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:40613
2024-11-23T22:54:36,008 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:54:36,008 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:54:36,008 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:54:36,009 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,39775,1732402475383 with port=40613, startcode=1732402475550
2024-11-23T22:54:36,009 DEBUG [RS:0;dd96e8dc8369:40613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:54:36,011 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:54:36,011 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,011 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39775 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,013 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e
2024-11-23T22:54:36,013 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33917
2024-11-23T22:54:36,013 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:54:36,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:54:36,026 DEBUG [RS:0;dd96e8dc8369:40613 {}] zookeeper.ZKUtil(111): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,026 WARN  [RS:0;dd96e8dc8369:40613 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:54:36,027 INFO  [RS:0;dd96e8dc8369:40613 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:54:36,027 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,027 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,40613,1732402475550]
2024-11-23T22:54:36,030 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:54:36,032 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:54:36,033 INFO  [RS:0;dd96e8dc8369:40613 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:54:36,033 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,033 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:54:36,034 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:54:36,034 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,034 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:54:36,035 DEBUG [RS:0;dd96e8dc8369:40613 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,037 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,40613,1732402475550-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:54:36,053 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:54:36,053 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,40613,1732402475550-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,053 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,054 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.Replication(171): dd96e8dc8369,40613,1732402475550 started
2024-11-23T22:54:36,071 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,071 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,40613,1732402475550, RpcServer on dd96e8dc8369/172.17.0.2:40613, sessionid=0x10169e7ed0c0001
2024-11-23T22:54:36,071 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:54:36,071 DEBUG [RS:0;dd96e8dc8369:40613 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,071 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,40613,1732402475550'
2024-11-23T22:54:36,072 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,40613,1732402475550'
2024-11-23T22:54:36,073 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:54:36,074 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:54:36,074 DEBUG [RS:0;dd96e8dc8369:40613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:54:36,074 INFO  [RS:0;dd96e8dc8369:40613 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:54:36,074 INFO  [RS:0;dd96e8dc8369:40613 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:54:36,112 WARN  [dd96e8dc8369:39775 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:54:36,168 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:36,176 INFO  [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C40613%2C1732402475550, suffix=, logDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550, archiveDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs, maxLogs=32
2024-11-23T22:54:36,177 INFO  [RS:0;dd96e8dc8369:40613 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40613%2C1732402475550.1732402476177
2024-11-23T22:54:36,182 INFO  [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402476177
2024-11-23T22:54:36,183 DEBUG [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34649:34649),(127.0.0.1/127.0.0.1:39777:39777)]
2024-11-23T22:54:36,363 DEBUG [dd96e8dc8369:39775 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:54:36,363 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,364 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,40613,1732402475550, state=OPENING
2024-11-23T22:54:36,373 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:54:36,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:36,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:54:36,384 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:54:36,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:36,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:36,384 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40613,1732402475550}]
2024-11-23T22:54:36,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:36,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:36,538 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:54:36,540 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:54:36,544 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:54:36,544 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:54:36,545 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C40613%2C1732402475550.meta, suffix=.meta, logDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550, archiveDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs, maxLogs=32
2024-11-23T22:54:36,546 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40613%2C1732402475550.meta.1732402476546.meta
2024-11-23T22:54:36,553 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.meta.1732402476546.meta
2024-11-23T22:54:36,556 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34649:34649),(127.0.0.1/127.0.0.1:39777:39777)]
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:54:36,557 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:54:36,557 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:54:36,559 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:54:36,560 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:54:36,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:36,561 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:36,561 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:54:36,561 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:54:36,561 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:36,562 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:36,562 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:54:36,562 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:54:36,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:36,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:36,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:54:36,563 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:54:36,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:36,564 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:54:36,564 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:54:36,565 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740
2024-11-23T22:54:36,566 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740
2024-11-23T22:54:36,567 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:54:36,567 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:54:36,567 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:54:36,568 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:54:36,569 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=880171, jitterRate=0.11919574439525604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:54:36,569 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:54:36,570 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402476558Writing region info on filesystem at 1732402476558Initializing all the Stores at 1732402476559 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402476559Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402476559Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402476559Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402476559Cleaning up temporary data from old regions at 1732402476567 (+8 ms)Running coprocessor post-open hooks at 1732402476569 (+2 ms)Region opened successfully at 1732402476570 (+1 ms)
2024-11-23T22:54:36,571 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402476538
2024-11-23T22:54:36,574 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:54:36,574 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:54:36,574 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,575 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,40613,1732402475550, state=OPEN
2024-11-23T22:54:36,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:54:36,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:54:36,617 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:36,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:54:36,620 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:54:36,620 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,40613,1732402475550 in 233 msec
2024-11-23T22:54:36,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:54:36,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 660 msec
2024-11-23T22:54:36,624 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:54:36,624 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:54:36,625 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:54:36,625 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,40613,1732402475550, seqNum=-1]
2024-11-23T22:54:36,626 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:54:36,627 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41307, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:54:36,633 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 714 msec
2024-11-23T22:54:36,633 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402476633, completionTime=-1
2024-11-23T22:54:36,633 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:54:36,633 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:54:36,635 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:54:36,635 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402536635
2024-11-23T22:54:36,635 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402596635
2024-11-23T22:54:36,635 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:39775, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,636 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,638 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.025sec
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:54:36,641 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:54:36,644 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:54:36,644 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:54:36,644 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39775,1732402475383-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:54:36,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ad6d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:36,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,39775,-1 for getting cluster id
2024-11-23T22:54:36,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:54:36,677 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df673e81-8e0a-4e6a-8504-9551941f2b6c'
2024-11-23T22:54:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:54:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df673e81-8e0a-4e6a-8504-9551941f2b6c"

2024-11-23T22:54:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ddfa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,39775,-1]
2024-11-23T22:54:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:54:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:54:36,681 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:54:36,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781dc17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:54:36,683 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:54:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,40613,1732402475550, seqNum=-1]
2024-11-23T22:54:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:54:36,686 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:54:36,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:36,689 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:54:36,692 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:54:36,693 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-23T22:54:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,39775,1732402475383
2024-11-23T22:54:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33e3b916
2024-11-23T22:54:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-23T22:54:36,696 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-23T22:54:36,697 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-23T22:54:36,697 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-23T22:54:36,697 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:54:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:36,701 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-23T22:54:36,701 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:36,701 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-23T22:54:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:54:36,702 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-23T22:54:36,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741835_1011 (size=405)
2024-11-23T22:54:36,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741835_1011 (size=405)
2024-11-23T22:54:36,713 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b1f1d9f1a8a87c2c251cd919ca1a7eb, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e
2024-11-23T22:54:36,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741836_1012 (size=88)
2024-11-23T22:54:36,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741836_1012 (size=88)
2024-11-23T22:54:36,722 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:36,723 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 8b1f1d9f1a8a87c2c251cd919ca1a7eb, disabling compactions & flushes
2024-11-23T22:54:36,723 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:36,723 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:36,723 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb. after waiting 0 ms
2024-11-23T22:54:36,723 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:36,723 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:36,723 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:
Waiting for close lock at 1732402476723Disabling compacts and flushes for region at 1732402476723Disabling writes for close at 1732402476723Writing region close event to WAL at 1732402476723Closed at 1732402476723
2024-11-23T22:54:36,724 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-23T22:54:36,725 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732402476724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402476724"}]},"ts":"1732402476724"}
2024-11-23T22:54:36,727 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-23T22:54:36,728 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-23T22:54:36,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402476728"}]},"ts":"1732402476728"}
2024-11-23T22:54:36,731 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-23T22:54:36,731 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=8b1f1d9f1a8a87c2c251cd919ca1a7eb, ASSIGN}]
2024-11-23T22:54:36,733 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=8b1f1d9f1a8a87c2c251cd919ca1a7eb, ASSIGN
2024-11-23T22:54:36,734 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=8b1f1d9f1a8a87c2c251cd919ca1a7eb, ASSIGN; state=OFFLINE, location=dd96e8dc8369,40613,1732402475550; forceNewPlan=false, retain=false
2024-11-23T22:54:36,885 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b1f1d9f1a8a87c2c251cd919ca1a7eb, regionState=OPENING, regionLocation=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:36,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=8b1f1d9f1a8a87c2c251cd919ca1a7eb, ASSIGN because future has completed
2024-11-23T22:54:36,892 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b1f1d9f1a8a87c2c251cd919ca1a7eb, server=dd96e8dc8369,40613,1732402475550}]
2024-11-23T22:54:37,055 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:37,055 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b1f1d9f1a8a87c2c251cd919ca1a7eb, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:54:37,056 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,056 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:54:37,056 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,056 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,059 INFO  [StoreOpener-8b1f1d9f1a8a87c2c251cd919ca1a7eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8b1f1d9f1a8a87c2c251cd919ca1a7eb 
2024-11-23T22:54:37,062 INFO  [StoreOpener-8b1f1d9f1a8a87c2c251cd919ca1a7eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b1f1d9f1a8a87c2c251cd919ca1a7eb columnFamilyName info
2024-11-23T22:54:37,062 DEBUG [StoreOpener-8b1f1d9f1a8a87c2c251cd919ca1a7eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:54:37,063 INFO  [StoreOpener-8b1f1d9f1a8a87c2c251cd919ca1a7eb-1 {}] regionserver.HStore(327): Store=8b1f1d9f1a8a87c2c251cd919ca1a7eb/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:54:37,063 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,064 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,064 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,065 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,065 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,066 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,068 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:54:37,069 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8b1f1d9f1a8a87c2c251cd919ca1a7eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=720200, jitterRate=-0.08421863615512848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:54:37,069 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:54:37,069 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:
Running coprocessor pre-open hook at 1732402477056Writing region info on filesystem at 1732402477056Initializing all the Stores at 1732402477058 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402477058Cleaning up temporary data from old regions at 1732402477065 (+7 ms)Running coprocessor post-open hooks at 1732402477069 (+4 ms)Region opened successfully at 1732402477069
2024-11-23T22:54:37,070 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb., pid=6, masterSystemTime=1732402477047
2024-11-23T22:54:37,073 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:37,073 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:37,074 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b1f1d9f1a8a87c2c251cd919ca1a7eb, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,40613,1732402475550
2024-11-23T22:54:37,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b1f1d9f1a8a87c2c251cd919ca1a7eb, server=dd96e8dc8369,40613,1732402475550 because future has completed
2024-11-23T22:54:37,081 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-23T22:54:37,081 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b1f1d9f1a8a87c2c251cd919ca1a7eb, server=dd96e8dc8369,40613,1732402475550 in 186 msec
2024-11-23T22:54:37,084 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-23T22:54:37,084 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=8b1f1d9f1a8a87c2c251cd919ca1a7eb, ASSIGN in 350 msec
2024-11-23T22:54:37,085 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-23T22:54:37,085 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402477085"}]},"ts":"1732402477085"}
2024-11-23T22:54:37,087 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-23T22:54:37,088 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-23T22:54:37,090 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 391 msec
2024-11-23T22:54:37,169 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:37,415 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:54:37,416 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,417 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,417 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,417 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:37,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,443 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:37,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:38,169 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:38,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:38,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:39,170 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:39,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:39,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:40,171 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:40,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:40,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:41,173 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:41,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:41,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:41,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:54:41,913 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-23T22:54:41,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:54:41,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-23T22:54:41,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-23T22:54:41,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-23T22:54:41,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:41,914 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-23T22:54:42,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:54:42,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-23T22:54:42,174 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:42,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:42,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:42,559 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:54:42,561 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,562 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,562 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,562 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,563 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,563 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,592 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,592 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,592 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:42,594 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:54:43,175 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:43,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:43,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:44,176 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:44,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:44,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:45,177 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:45,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:45,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:46,179 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:46,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:46,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:54:46,726 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-23T22:54:46,726 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-23T22:54:46,729 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:46,729 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb., hostname=dd96e8dc8369,40613,1732402475550, seqNum=2]
2024-11-23T22:54:46,741 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:46,747 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-23T22:54:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-23T22:54:46,749 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-23T22:54:46,750 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-23T22:54:46,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-23T22:54:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:46,914 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:54:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/90e0868ea2494a96991f0ad927b48f81 is 1080, key is row0001/info:/1732402486734/Put/seqid=0
2024-11-23T22:54:46,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741837_1013 (size=6033)
2024-11-23T22:54:46,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741837_1013 (size=6033)
2024-11-23T22:54:47,180 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:47,346 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/90e0868ea2494a96991f0ad927b48f81
2024-11-23T22:54:47,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/90e0868ea2494a96991f0ad927b48f81 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81
2024-11-23T22:54:47,364 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81, entries=1, sequenceid=5, filesize=5.9 K
2024-11-23T22:54:47,366 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 451ms, sequenceid=5, compaction requested=false
2024-11-23T22:54:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:

2024-11-23T22:54:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:47,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-23T22:54:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-23T22:54:47,375 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-23T22:54:47,375 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec
2024-11-23T22:54:47,379 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 633 msec
2024-11-23T22:54:47,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:47,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:48,180 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:48,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:48,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:49,181 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:49,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:49,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:50,182 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:50,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:50,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:51,182 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:51,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:51,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:52,183 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:52,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:52,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:53,184 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:53,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:53,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251 after 68064ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:53,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:53,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta after 68061ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-23T22:54:54,184 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:54,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:54,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:55,185 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:55,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:55,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:56,186 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:56,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:56,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-23T22:54:56,786 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-23T22:54:56,792 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:54:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-23T22:54:56,796 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-23T22:54:56,797 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-23T22:54:56,798 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-23T22:54:56,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-23T22:54:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:56,952 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:54:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/2590a124fd944cf68ea76738cba8655a is 1080, key is row0002/info:/1732402496789/Put/seqid=0
2024-11-23T22:54:56,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741838_1014 (size=6033)
2024-11-23T22:54:56,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741838_1014 (size=6033)
2024-11-23T22:54:56,962 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/2590a124fd944cf68ea76738cba8655a
2024-11-23T22:54:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/2590a124fd944cf68ea76738cba8655a as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a
2024-11-23T22:54:56,976 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a, entries=1, sequenceid=9, filesize=5.9 K
2024-11-23T22:54:56,977 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 25ms, sequenceid=9, compaction requested=false
2024-11-23T22:54:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:

2024-11-23T22:54:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:54:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-23T22:54:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-23T22:54:56,982 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-23T22:54:56,982 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-11-23T22:54:56,984 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 191 msec
2024-11-23T22:54:57,187 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:57,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:57,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:58,187 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:58,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:58,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:59,188 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:59,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:54:59,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:00,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:00,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:00,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:01,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:01,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:01,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:02,190 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:02,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:02,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:03,191 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:03,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:03,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:04,192 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:04,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:04,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:05,193 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:05,333 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:55:05,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:05,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:06,193 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:06,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:06,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-23T22:55:06,876 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-23T22:55:06,878 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40613%2C1732402475550.1732402506878
2024-11-23T22:55:06,884 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:06,884 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:06,884 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:06,884 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:06,885 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:06,885 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402476177 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402506878
2024-11-23T22:55:06,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39777:39777),(127.0.0.1/127.0.0.1:34649:34649)]
2024-11-23T22:55:06,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402476177 is not closed yet, will try archiving it next time
2024-11-23T22:55:06,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741833_1009 (size=5546)
2024-11-23T22:55:06,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741833_1009 (size=5546)
2024-11-23T22:55:06,887 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:55:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:55:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-23T22:55:06,890 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-23T22:55:06,891 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-23T22:55:06,892 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-23T22:55:07,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-23T22:55:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:07,046 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:55:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/c229a074f0114a3598708e7acc144462 is 1080, key is row0003/info:/1732402506877/Put/seqid=0
2024-11-23T22:55:07,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741840_1016 (size=6033)
2024-11-23T22:55:07,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741840_1016 (size=6033)
2024-11-23T22:55:07,059 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/c229a074f0114a3598708e7acc144462
2024-11-23T22:55:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/c229a074f0114a3598708e7acc144462 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462
2024-11-23T22:55:07,073 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462, entries=1, sequenceid=13, filesize=5.9 K
2024-11-23T22:55:07,074 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 28ms, sequenceid=13, compaction requested=true
2024-11-23T22:55:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:

2024-11-23T22:55:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-23T22:55:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-23T22:55:07,079 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-23T22:55:07,079 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 186 msec
2024-11-23T22:55:07,082 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 193 msec
2024-11-23T22:55:07,194 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:07,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:07,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:08,195 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:08,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:08,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:09,195 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:09,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:09,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:10,196 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:10,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:10,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:11,197 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:11,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:11,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:12,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:12,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:12,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:13,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:13,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:13,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:14,199 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:14,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:14,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:15,200 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:15,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:15,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:16,200 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:16,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:16,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:16,644 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-23T22:55:16,644 INFO  [master/dd96e8dc8369:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-23T22:55:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-23T22:55:16,976 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-23T22:55:16,976 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:16,977 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:55:16,977 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info is initiating minor compaction (all files)
2024-11-23T22:55:16,977 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:55:16,977 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:16,977 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:16,978 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462] into tmpdir=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp, totalSize=17.7 K
2024-11-23T22:55:16,978 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 90e0868ea2494a96991f0ad927b48f81, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732402486734
2024-11-23T22:55:16,978 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2590a124fd944cf68ea76738cba8655a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732402496789
2024-11-23T22:55:16,979 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c229a074f0114a3598708e7acc144462, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732402506877
2024-11-23T22:55:16,991 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8b1f1d9f1a8a87c2c251cd919ca1a7eb#info#compaction#47 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:16,992 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/e802766029164a5eb8f6ba1d218b884f is 1080, key is row0001/info:/1732402486734/Put/seqid=0
2024-11-23T22:55:16,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741841_1017 (size=8296)
2024-11-23T22:55:16,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741841_1017 (size=8296)
2024-11-23T22:55:17,003 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/e802766029164a5eb8f6ba1d218b884f as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/e802766029164a5eb8f6ba1d218b884f
2024-11-23T22:55:17,011 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info of 8b1f1d9f1a8a87c2c251cd919ca1a7eb into e802766029164a5eb8f6ba1d218b884f(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:17,011 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:

2024-11-23T22:55:17,014 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40613%2C1732402475550.1732402517014
2024-11-23T22:55:17,019 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:17,020 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:17,020 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:17,020 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:17,020 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:17,020 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402506878 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402517014
2024-11-23T22:55:17,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741839_1015 (size=2520)
2024-11-23T22:55:17,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741839_1015 (size=2520)
2024-11-23T22:55:17,024 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402476177 to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs/dd96e8dc8369%2C40613%2C1732402475550.1732402476177
2024-11-23T22:55:17,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34649:34649),(127.0.0.1/127.0.0.1:39777:39777)]
2024-11-23T22:55:17,025 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:55:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:55:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-23T22:55:17,028 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-23T22:55:17,029 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-23T22:55:17,029 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-23T22:55:17,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-23T22:55:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:17,183 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:55:17,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/0f71aa6bd533488c83db7745a865d86e is 1080, key is row0000/info:/1732402517012/Put/seqid=0
2024-11-23T22:55:17,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741843_1019 (size=6033)
2024-11-23T22:55:17,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741843_1019 (size=6033)
2024-11-23T22:55:17,195 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/0f71aa6bd533488c83db7745a865d86e
2024-11-23T22:55:17,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/0f71aa6bd533488c83db7745a865d86e as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/0f71aa6bd533488c83db7745a865d86e
2024-11-23T22:55:17,208 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/0f71aa6bd533488c83db7745a865d86e, entries=1, sequenceid=18, filesize=5.9 K
2024-11-23T22:55:17,209 INFO  [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 26ms, sequenceid=18, compaction requested=false
2024-11-23T22:55:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:

2024-11-23T22:55:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-23T22:55:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-23T22:55:17,215 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-23T22:55:17,215 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec
2024-11-23T22:55:17,218 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 191 msec
2024-11-23T22:55:17,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:17,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:18,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:18,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:18,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:19,202 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:19,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:19,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:20,203 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:20,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:20,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:21,203 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:21,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:21,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:22,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8b1f1d9f1a8a87c2c251cd919ca1a7eb, had cached 0 bytes from a total of 14329
2024-11-23T22:55:22,204 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:22,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:22,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:23,205 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:23,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:23,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:24,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:24,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:24,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:25,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:25,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:25,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:26,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:26,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:26,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-23T22:55:27,106 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-23T22:55:27,110 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C40613%2C1732402475550.1732402527110
2024-11-23T22:55:27,117 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,117 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,117 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,117 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,117 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,118 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402517014 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402527110
2024-11-23T22:55:27,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34649:34649),(127.0.0.1/127.0.0.1:39777:39777)]
2024-11-23T22:55:27,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402517014 is not closed yet, will try archiving it next time
2024-11-23T22:55:27,119 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402506878 to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs/dd96e8dc8369%2C40613%2C1732402475550.1732402506878
2024-11-23T22:55:27,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:55:27,119 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:55:27,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:55:27,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:27,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:27,119 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:55:27,119 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:55:27,119 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=871722539, stopped=false
2024-11-23T22:55:27,120 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,39775,1732402475383
2024-11-23T22:55:27,120 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741842_1018 (size=2026)
2024-11-23T22:55:27,120 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741842_1018 (size=2026)
2024-11-23T22:55:27,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:55:27,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:55:27,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:27,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:27,159 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:55:27,159 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:55:27,159 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:55:27,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:27,159 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,40613,1732402475550' *****
2024-11-23T22:55:27,160 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:55:27,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:55:27,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,40613,1732402475550
2024-11-23T22:55:27,160 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:55:27,160 INFO  [RS:0;dd96e8dc8369:40613 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:40613.
2024-11-23T22:55:27,160 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b1f1d9f1a8a87c2c251cd919ca1a7eb, disabling compactions & flushes
2024-11-23T22:55:27,160 DEBUG [RS:0;dd96e8dc8369:40613 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:55:27,160 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:27,160 DEBUG [RS:0;dd96e8dc8369:40613 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:27,160 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb. after waiting 1 ms
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:27,161 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:55:27,161 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:55:27,161 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:55:27,161 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:55:27,161 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:55:27,161 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-23T22:55:27,161 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1325): Online Regions={8b1f1d9f1a8a87c2c251cd919ca1a7eb=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb., 1588230740=hbase:meta,,1.1588230740}
2024-11-23T22:55:27,161 DEBUG [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8b1f1d9f1a8a87c2c251cd919ca1a7eb
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:55:27,161 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:55:27,161 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:55:27,161 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-23T22:55:27,165 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/7023f42e55c5410bb902497ae62cecdd is 1080, key is row0001/info:/1732402527108/Put/seqid=0
2024-11-23T22:55:27,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741845_1021 (size=6033)
2024-11-23T22:55:27,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741845_1021 (size=6033)
2024-11-23T22:55:27,171 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/7023f42e55c5410bb902497ae62cecdd
2024-11-23T22:55:27,182 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/.tmp/info/7023f42e55c5410bb902497ae62cecdd as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/7023f42e55c5410bb902497ae62cecdd
2024-11-23T22:55:27,184 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/info/7eaf622dfdd04d1da88aac178d7427dc is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb./info:regioninfo/1732402477073/Put/seqid=0
2024-11-23T22:55:27,188 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/7023f42e55c5410bb902497ae62cecdd, entries=1, sequenceid=22, filesize=5.9 K
2024-11-23T22:55:27,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741846_1022 (size=7308)
2024-11-23T22:55:27,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741846_1022 (size=7308)
2024-11-23T22:55:27,189 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/info/7eaf622dfdd04d1da88aac178d7427dc
2024-11-23T22:55:27,189 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 28ms, sequenceid=22, compaction requested=true
2024-11-23T22:55:27,190 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462] to archive
2024-11-23T22:55:27,190 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:55:27,192 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81 to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/90e0868ea2494a96991f0ad927b48f81
2024-11-23T22:55:27,193 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/2590a124fd944cf68ea76738cba8655a
2024-11-23T22:55:27,195 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462 to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/info/c229a074f0114a3598708e7acc144462
2024-11-23T22:55:27,195 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=dd96e8dc8369:39775 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-23T22:55:27,196 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [90e0868ea2494a96991f0ad927b48f81=6033, 2590a124fd944cf68ea76738cba8655a=6033, c229a074f0114a3598708e7acc144462=6033]
2024-11-23T22:55:27,199 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/8b1f1d9f1a8a87c2c251cd919ca1a7eb/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-23T22:55:27,200 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:27,200 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b1f1d9f1a8a87c2c251cd919ca1a7eb:
Waiting for close lock at 1732402527160Running coprocessor pre-close hooks at 1732402527160Disabling compacts and flushes for region at 1732402527160Disabling writes for close at 1732402527161 (+1 ms)Obtaining lock to block concurrent updates at 1732402527161Preparing flush snapshotting stores in 8b1f1d9f1a8a87c2c251cd919ca1a7eb at 1732402527161Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732402527161Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb. at 1732402527162 (+1 ms)Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info: creating writer at 1732402527162Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info: appending metadata at 1732402527165 (+3 ms)Flushing 8b1f1d9f1a8a87c2c251cd919ca1a7eb/info: closing flushed file at 1732402527165Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7999a46b: reopening flushed file at 1732402527181 (+16 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 8b1f1d9f1a8a87c2c251cd919ca1a7eb in 28ms, sequenceid=22, compaction requested=true at 1732402527189 (+8 ms)Writing region close event to WAL at 1732402527196 (+7 ms)Running coprocessor post-close hooks at 1732402527200 (+4 ms)Closed at 1732402527200
2024-11-23T22:55:27,200 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732402476697.8b1f1d9f1a8a87c2c251cd919ca1a7eb.
2024-11-23T22:55:27,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:27,211 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/ns/b2f85894c86e4bb19431cfc21fb46e56 is 43, key is default/ns:d/1732402476627/Put/seqid=0
2024-11-23T22:55:27,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741847_1023 (size=5153)
2024-11-23T22:55:27,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741847_1023 (size=5153)
2024-11-23T22:55:27,216 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/ns/b2f85894c86e4bb19431cfc21fb46e56
2024-11-23T22:55:27,236 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/table/1f8a8d72abcc4216ba566edfa52d66d2 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732402477085/Put/seqid=0
2024-11-23T22:55:27,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741848_1024 (size=5508)
2024-11-23T22:55:27,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741848_1024 (size=5508)
2024-11-23T22:55:27,242 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/table/1f8a8d72abcc4216ba566edfa52d66d2
2024-11-23T22:55:27,248 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/info/7eaf622dfdd04d1da88aac178d7427dc as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/info/7eaf622dfdd04d1da88aac178d7427dc
2024-11-23T22:55:27,253 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/info/7eaf622dfdd04d1da88aac178d7427dc, entries=10, sequenceid=11, filesize=7.1 K
2024-11-23T22:55:27,254 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/ns/b2f85894c86e4bb19431cfc21fb46e56 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/ns/b2f85894c86e4bb19431cfc21fb46e56
2024-11-23T22:55:27,259 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/ns/b2f85894c86e4bb19431cfc21fb46e56, entries=2, sequenceid=11, filesize=5.0 K
2024-11-23T22:55:27,260 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/.tmp/table/1f8a8d72abcc4216ba566edfa52d66d2 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/table/1f8a8d72abcc4216ba566edfa52d66d2
2024-11-23T22:55:27,267 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/table/1f8a8d72abcc4216ba566edfa52d66d2, entries=2, sequenceid=11, filesize=5.4 K
2024-11-23T22:55:27,268 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=11, compaction requested=false
2024-11-23T22:55:27,273 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-23T22:55:27,273 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:55:27,274 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:55:27,274 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402527161Running coprocessor pre-close hooks at 1732402527161Disabling compacts and flushes for region at 1732402527161Disabling writes for close at 1732402527161Obtaining lock to block concurrent updates at 1732402527161Preparing flush snapshotting stores in 1588230740 at 1732402527161Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732402527161Flushing stores of hbase:meta,,1.1588230740 at 1732402527162 (+1 ms)Flushing 1588230740/info: creating writer at 1732402527162Flushing 1588230740/info: appending metadata at 1732402527184 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732402527184Flushing 1588230740/ns: creating writer at 1732402527195 (+11 ms)Flushing 1588230740/ns: appending metadata at 1732402527211 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732402527211Flushing 1588230740/table: creating writer at 1732402527221 (+10 ms)Flushing 1588230740/table: appending metadata at 1732402527236 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732402527236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ff3d2ce: reopening flushed file at 1732402527247 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e2632d: reopening flushed file at 1732402527253 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a36419a: reopening flushed file at 1732402527260 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=11, compaction requested=false at 1732402527268 (+8 ms)Writing region close event to WAL at 1732402527269 (+1 ms)Running coprocessor post-close hooks at 1732402527273 (+4 ms)Closed at 1732402527274 (+1 ms)
2024-11-23T22:55:27,274 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:55:27,361 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,40613,1732402475550; all regions closed.
2024-11-23T22:55:27,362 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,362 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,362 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,362 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,362 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741834_1010 (size=3306)
2024-11-23T22:55:27,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741834_1010 (size=3306)
2024-11-23T22:55:27,368 DEBUG [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs
2024-11-23T22:55:27,368 INFO  [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C40613%2C1732402475550.meta:.meta(num 1732402476546)
2024-11-23T22:55:27,368 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,368 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,368 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,369 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,369 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741844_1020 (size=1252)
2024-11-23T22:55:27,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741844_1020 (size=1252)
2024-11-23T22:55:27,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:27,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:27,521 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/WALs/dd96e8dc8369,40613,1732402475550/dd96e8dc8369%2C40613%2C1732402475550.1732402517014 to hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs/dd96e8dc8369%2C40613%2C1732402475550.1732402517014
2024-11-23T22:55:27,523 DEBUG [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/oldWALs
2024-11-23T22:55:27,523 INFO  [RS:0;dd96e8dc8369:40613 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C40613%2C1732402475550:(num 1732402527110)
2024-11-23T22:55:27,523 DEBUG [RS:0;dd96e8dc8369:40613 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:27,524 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:55:27,524 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:55:27,524 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:55:27,524 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:55:27,524 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:55:27,524 INFO  [RS:0;dd96e8dc8369:40613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40613
2024-11-23T22:55:27,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:55:27,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,40613,1732402475550
2024-11-23T22:55:27,576 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:55:27,586 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,40613,1732402475550]
2024-11-23T22:55:27,596 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,40613,1732402475550 already deleted, retry=false
2024-11-23T22:55:27,596 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,40613,1732402475550 expired; onlineServers=0
2024-11-23T22:55:27,596 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,39775,1732402475383' *****
2024-11-23T22:55:27,596 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:55:27,597 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:55:27,597 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:55:27,597 DEBUG [M:0;dd96e8dc8369:39775 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:55:27,597 DEBUG [M:0;dd96e8dc8369:39775 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:55:27,597 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:55:27,597 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402475923 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402475923,5,FailOnTimeoutGroup]
2024-11-23T22:55:27,597 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402475924 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402475924,5,FailOnTimeoutGroup]
2024-11-23T22:55:27,597 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:55:27,597 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:55:27,597 DEBUG [M:0;dd96e8dc8369:39775 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:55:27,597 INFO  [M:0;dd96e8dc8369:39775 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:55:27,598 INFO  [M:0;dd96e8dc8369:39775 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:55:27,598 INFO  [M:0;dd96e8dc8369:39775 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:55:27,598 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:55:27,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:55:27,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:27,607 DEBUG [M:0;dd96e8dc8369:39775 {}] zookeeper.ZKUtil(347): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:55:27,607 WARN  [M:0;dd96e8dc8369:39775 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:55:27,607 INFO  [M:0;dd96e8dc8369:39775 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/.lastflushedseqids
2024-11-23T22:55:27,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741849_1025 (size=130)
2024-11-23T22:55:27,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741849_1025 (size=130)
2024-11-23T22:55:27,614 INFO  [M:0;dd96e8dc8369:39775 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:55:27,614 INFO  [M:0;dd96e8dc8369:39775 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:55:27,614 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:55:27,614 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:27,614 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:27,614 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:55:27,614 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:27,614 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.46 KB heapSize=54.85 KB
2024-11-23T22:55:27,631 DEBUG [M:0;dd96e8dc8369:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/135118a138924083808e6c582720afab is 82, key is hbase:meta,,1/info:regioninfo/1732402476574/Put/seqid=0
2024-11-23T22:55:27,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741850_1026 (size=5672)
2024-11-23T22:55:27,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741850_1026 (size=5672)
2024-11-23T22:55:27,635 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/135118a138924083808e6c582720afab
2024-11-23T22:55:27,656 DEBUG [M:0;dd96e8dc8369:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1aa17c48f4934a84a711aac7adea2673 is 795, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402477089/Put/seqid=0
2024-11-23T22:55:27,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741851_1027 (size=7809)
2024-11-23T22:55:27,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741851_1027 (size=7809)
2024-11-23T22:55:27,661 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.86 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1aa17c48f4934a84a711aac7adea2673
2024-11-23T22:55:27,665 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1aa17c48f4934a84a711aac7adea2673
2024-11-23T22:55:27,683 DEBUG [M:0;dd96e8dc8369:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a490c2fc7c24073ace05431ded94758 is 69, key is dd96e8dc8369,40613,1732402475550/rs:state/1732402476012/Put/seqid=0
2024-11-23T22:55:27,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:55:27,686 INFO  [RS:0;dd96e8dc8369:40613 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:55:27,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40613-0x10169e7ed0c0001, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:55:27,686 INFO  [RS:0;dd96e8dc8369:40613 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,40613,1732402475550; zookeeper connection closed.
2024-11-23T22:55:27,686 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3424480a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3424480a
2024-11-23T22:55:27,687 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:55:27,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741852_1028 (size=5156)
2024-11-23T22:55:27,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741852_1028 (size=5156)
2024-11-23T22:55:27,687 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a490c2fc7c24073ace05431ded94758
2024-11-23T22:55:27,707 DEBUG [M:0;dd96e8dc8369:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a619de7b2b441488b1a73461f9847c5 is 52, key is load_balancer_on/state:d/1732402476691/Put/seqid=0
2024-11-23T22:55:27,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741853_1029 (size=5056)
2024-11-23T22:55:27,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741853_1029 (size=5056)
2024-11-23T22:55:27,712 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a619de7b2b441488b1a73461f9847c5
2024-11-23T22:55:27,717 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/135118a138924083808e6c582720afab as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/135118a138924083808e6c582720afab
2024-11-23T22:55:27,722 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/135118a138924083808e6c582720afab, entries=8, sequenceid=121, filesize=5.5 K
2024-11-23T22:55:27,723 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1aa17c48f4934a84a711aac7adea2673 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1aa17c48f4934a84a711aac7adea2673
2024-11-23T22:55:27,728 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1aa17c48f4934a84a711aac7adea2673
2024-11-23T22:55:27,728 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1aa17c48f4934a84a711aac7adea2673, entries=14, sequenceid=121, filesize=7.6 K
2024-11-23T22:55:27,729 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a490c2fc7c24073ace05431ded94758 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a490c2fc7c24073ace05431ded94758
2024-11-23T22:55:27,735 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a490c2fc7c24073ace05431ded94758, entries=1, sequenceid=121, filesize=5.0 K
2024-11-23T22:55:27,736 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a619de7b2b441488b1a73461f9847c5 as hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2a619de7b2b441488b1a73461f9847c5
2024-11-23T22:55:27,741 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33917/user/jenkins/test-data/daa8e955-f22c-aef1-bd5f-0efaddda6b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2a619de7b2b441488b1a73461f9847c5, entries=1, sequenceid=121, filesize=4.9 K
2024-11-23T22:55:27,742 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.46 KB/44503, heapSize ~54.79 KB/56104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=121, compaction requested=false
2024-11-23T22:55:27,744 INFO  [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:27,744 DEBUG [M:0;dd96e8dc8369:39775 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402527614Disabling compacts and flushes for region at 1732402527614Disabling writes for close at 1732402527614Obtaining lock to block concurrent updates at 1732402527614Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402527614Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44503, getHeapSize=56104, getOffHeapSize=0, getCellsCount=140 at 1732402527615 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402527616 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402527616Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402527630 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402527630Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402527640 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402527655 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402527655Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402527665 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402527682 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402527682Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402527692 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402527706 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402527706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65440304: reopening flushed file at 1732402527716 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41faa4f6: reopening flushed file at 1732402527723 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e69b7: reopening flushed file at 1732402527728 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@781bc3c2: reopening flushed file at 1732402527735 (+7 ms)Finished flush of dataSize ~43.46 KB/44503, heapSize ~54.79 KB/56104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=121, compaction requested=false at 1732402527742 (+7 ms)Writing region close event to WAL at 1732402527744 (+2 ms)Closed at 1732402527744
2024-11-23T22:55:27,744 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,744 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,744 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,744 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,745 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:55:27,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36151 is added to blk_1073741830_1006 (size=52900)
2024-11-23T22:55:27,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37707 is added to blk_1073741830_1006 (size=52900)
2024-11-23T22:55:27,747 INFO  [M:0;dd96e8dc8369:39775 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:55:27,747 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:55:27,747 INFO  [M:0;dd96e8dc8369:39775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39775
2024-11-23T22:55:27,747 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:55:27,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:55:27,859 INFO  [M:0;dd96e8dc8369:39775 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:55:27,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x10169e7ed0c0000, quorum=127.0.0.1:59470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:55:27,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cb8e2e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:55:27,888 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33803ea9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:55:27,888 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:55:27,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@718e5030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:55:27,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@432bc536{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,STOPPED}
2024-11-23T22:55:27,890 WARN  [BP-2045060364-172.17.0.2-1732402473324 heartbeating to localhost/127.0.0.1:33917 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:55:27,890 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:55:27,890 WARN  [BP-2045060364-172.17.0.2-1732402473324 heartbeating to localhost/127.0.0.1:33917 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2045060364-172.17.0.2-1732402473324 (Datanode Uuid f12b4171-7ce1-4a14-8a97-7463c2628d64) service to localhost/127.0.0.1:33917
2024-11-23T22:55:27,890 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:55:27,890 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data3/current/BP-2045060364-172.17.0.2-1732402473324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:55:27,891 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data4/current/BP-2045060364-172.17.0.2-1732402473324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:55:27,891 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:55:27,893 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cda66bc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:55:27,893 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e84571d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:55:27,893 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:55:27,893 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@188d3049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:55:27,894 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f81b4d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,STOPPED}
2024-11-23T22:55:27,895 WARN  [BP-2045060364-172.17.0.2-1732402473324 heartbeating to localhost/127.0.0.1:33917 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:55:27,895 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:55:27,895 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:55:27,895 WARN  [BP-2045060364-172.17.0.2-1732402473324 heartbeating to localhost/127.0.0.1:33917 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2045060364-172.17.0.2-1732402473324 (Datanode Uuid 3a61fb35-f0c7-47cf-b0c4-7222b1f197c6) service to localhost/127.0.0.1:33917
2024-11-23T22:55:27,896 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data1/current/BP-2045060364-172.17.0.2-1732402473324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:55:27,896 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/cluster_1391a6ab-adf8-a600-e0e6-1065b0f52bdc/data/data2/current/BP-2045060364-172.17.0.2-1732402473324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:55:27,896 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:55:27,902 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58d2b6e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:55:27,902 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14642123{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:55:27,902 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:55:27,902 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e264d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:55:27,902 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38b36302{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir/,STOPPED}
2024-11-23T22:55:27,908 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:55:27,931 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:55:27,939 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=209 (was 183)
Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:33917 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:33917 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33917
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33917
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33917
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:33917
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:33917 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: regionserver/dd96e8dc8369:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33917
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 106) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6471 (was 7000)
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=209, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=6471
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.log.dir so I do NOT create it in target/test-data/343f8187-b886-be49-2d81-2468d21421b0
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fd7495fe-44d9-caba-e522-6c3a687b6661/hadoop.tmp.dir so I do NOT create it in target/test-data/343f8187-b886-be49-2d81-2468d21421b0
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370, deleteOnExit=true
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:55:27,947 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/test.cache.data in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:55:27,948 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:55:27,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:55:27,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:55:27,963 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:55:28,041 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:55:28,208 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:28,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:28,475 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:55:28,478 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:55:28,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:55:28,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:55:28,480 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:55:28,480 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:55:28,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32fa876c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:55:28,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e27d2b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:55:28,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:28,583 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bb84093{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/java.io.tmpdir/jetty-localhost-33347-hadoop-hdfs-3_4_1-tests_jar-_-any-5149748291660931658/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:55:28,583 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49a552fc{HTTP/1.1, (http/1.1)}{localhost:33347}
2024-11-23T22:55:28,583 INFO  [Time-limited test {}] server.Server(415): Started @253908ms
2024-11-23T22:55:28,596 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:55:28,902 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:55:28,905 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:55:28,905 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:55:28,905 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:55:28,905 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:55:28,906 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7edf9828{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:55:28,906 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@98b6596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:55:29,013 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c8f3fae{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/java.io.tmpdir/jetty-localhost-37181-hadoop-hdfs-3_4_1-tests_jar-_-any-3108079464013895871/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:55:29,013 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43cde2d6{HTTP/1.1, (http/1.1)}{localhost:37181}
2024-11-23T22:55:29,013 INFO  [Time-limited test {}] server.Server(415): Started @254338ms
2024-11-23T22:55:29,014 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:55:29,041 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:55:29,044 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:55:29,044 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:55:29,044 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:55:29,045 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:55:29,045 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b9fec83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:55:29,045 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68b06d6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:55:29,155 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ec1477d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/java.io.tmpdir/jetty-localhost-41691-hadoop-hdfs-3_4_1-tests_jar-_-any-22788661312162814/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:55:29,156 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5169cded{HTTP/1.1, (http/1.1)}{localhost:41691}
2024-11-23T22:55:29,156 INFO  [Time-limited test {}] server.Server(415): Started @254481ms
2024-11-23T22:55:29,157 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:55:29,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:29,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:29,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:30,040 WARN  [Thread-1987 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data1/current/BP-2077918267-172.17.0.2-1732402527974/current, will proceed with Du for space computation calculation, 
2024-11-23T22:55:30,040 WARN  [Thread-1988 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data2/current/BP-2077918267-172.17.0.2-1732402527974/current, will proceed with Du for space computation calculation, 
2024-11-23T22:55:30,059 WARN  [Thread-1951 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:55:30,061 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4450b0e5639b6fbb with lease ID 0xd18b4d00bb42736c: Processing first storage report for DS-c3c0b39c-df9a-44a8-869f-4ed304e74b9d from datanode DatanodeRegistration(127.0.0.1:34353, datanodeUuid=b3db603e-c19e-4ee9-87b8-748baac1eec7, infoPort=37645, infoSecurePort=0, ipcPort=32899, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974)
2024-11-23T22:55:30,061 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4450b0e5639b6fbb with lease ID 0xd18b4d00bb42736c: from storage DS-c3c0b39c-df9a-44a8-869f-4ed304e74b9d node DatanodeRegistration(127.0.0.1:34353, datanodeUuid=b3db603e-c19e-4ee9-87b8-748baac1eec7, infoPort=37645, infoSecurePort=0, ipcPort=32899, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:55:30,061 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4450b0e5639b6fbb with lease ID 0xd18b4d00bb42736c: Processing first storage report for DS-8da5e6bf-c7f5-47de-8894-3cc2636f3c43 from datanode DatanodeRegistration(127.0.0.1:34353, datanodeUuid=b3db603e-c19e-4ee9-87b8-748baac1eec7, infoPort=37645, infoSecurePort=0, ipcPort=32899, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974)
2024-11-23T22:55:30,061 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4450b0e5639b6fbb with lease ID 0xd18b4d00bb42736c: from storage DS-8da5e6bf-c7f5-47de-8894-3cc2636f3c43 node DatanodeRegistration(127.0.0.1:34353, datanodeUuid=b3db603e-c19e-4ee9-87b8-748baac1eec7, infoPort=37645, infoSecurePort=0, ipcPort=32899, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:55:30,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:30,284 WARN  [Thread-1998 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data3/current/BP-2077918267-172.17.0.2-1732402527974/current, will proceed with Du for space computation calculation, 
2024-11-23T22:55:30,284 WARN  [Thread-1999 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data4/current/BP-2077918267-172.17.0.2-1732402527974/current, will proceed with Du for space computation calculation, 
2024-11-23T22:55:30,300 WARN  [Thread-1974 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:55:30,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99a73b6afc46ac1d with lease ID 0xd18b4d00bb42736d: Processing first storage report for DS-0d268b9e-0a71-472d-9a7d-36b78565ebf4 from datanode DatanodeRegistration(127.0.0.1:34181, datanodeUuid=e39ac941-01e7-46a1-b8f1-10c1ab73d3ba, infoPort=38537, infoSecurePort=0, ipcPort=46859, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974)
2024-11-23T22:55:30,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99a73b6afc46ac1d with lease ID 0xd18b4d00bb42736d: from storage DS-0d268b9e-0a71-472d-9a7d-36b78565ebf4 node DatanodeRegistration(127.0.0.1:34181, datanodeUuid=e39ac941-01e7-46a1-b8f1-10c1ab73d3ba, infoPort=38537, infoSecurePort=0, ipcPort=46859, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:55:30,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99a73b6afc46ac1d with lease ID 0xd18b4d00bb42736d: Processing first storage report for DS-adb15de7-1beb-460c-b893-4d9a62bb88a8 from datanode DatanodeRegistration(127.0.0.1:34181, datanodeUuid=e39ac941-01e7-46a1-b8f1-10c1ab73d3ba, infoPort=38537, infoSecurePort=0, ipcPort=46859, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974)
2024-11-23T22:55:30,302 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99a73b6afc46ac1d with lease ID 0xd18b4d00bb42736d: from storage DS-adb15de7-1beb-460c-b893-4d9a62bb88a8 node DatanodeRegistration(127.0.0.1:34181, datanodeUuid=e39ac941-01e7-46a1-b8f1-10c1ab73d3ba, infoPort=38537, infoSecurePort=0, ipcPort=46859, storageInfo=lv=-57;cid=testClusterID;nsid=87162242;c=1732402527974), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:55:30,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0
2024-11-23T22:55:30,391 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/zookeeper_0, clientPort=56865, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:55:30,392 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56865
2024-11-23T22:55:30,392 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,394 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:55:30,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:55:30,405 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392 with version=8
2024-11-23T22:55:30,405 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:55:30,408 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:55:30,409 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:55:30,410 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35201
2024-11-23T22:55:30,412 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35201 connecting to ZooKeeper ensemble=127.0.0.1:56865
2024-11-23T22:55:30,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:30,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:30,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352010x0, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:55:30,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35201-0x10169e8c4010000 connected
2024-11-23T22:55:30,580 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,582 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:55:30,584 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392, hbase.cluster.distributed=false
2024-11-23T22:55:30,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:55:30,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35201
2024-11-23T22:55:30,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35201
2024-11-23T22:55:30,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35201
2024-11-23T22:55:30,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35201
2024-11-23T22:55:30,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35201
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:55:30,608 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:55:30,609 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:55:30,609 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46461
2024-11-23T22:55:30,611 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46461 connecting to ZooKeeper ensemble=127.0.0.1:56865
2024-11-23T22:55:30,612 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,613 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464610x0, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:55:30,628 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464610x0, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:55:30,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46461-0x10169e8c4010001 connected
2024-11-23T22:55:30,629 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:55:30,629 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:55:30,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:55:30,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:55:30,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46461
2024-11-23T22:55:30,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46461
2024-11-23T22:55:30,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46461
2024-11-23T22:55:30,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46461
2024-11-23T22:55:30,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46461
2024-11-23T22:55:30,646 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:35201
2024-11-23T22:55:30,646 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:55:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:55:30,654 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:55:30,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,664 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:55:30,665 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,35201,1732402530408 from backup master directory
2024-11-23T22:55:30,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:55:30,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:55:30,738 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:55:30,738 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,742 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/hbase.id] with ID: 92c3a864-f3ac-44ea-a493-c92f62ff9095
2024-11-23T22:55:30,742 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/.tmp/hbase.id
2024-11-23T22:55:30,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:55:30,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:55:30,749 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/.tmp/hbase.id]:[hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/hbase.id]
2024-11-23T22:55:30,759 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:30,759 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:55:30,761 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-23T22:55:30,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:55:30,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:55:30,796 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:55:30,797 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:55:30,797 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:55:30,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:55:30,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:55:30,804 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store
2024-11-23T22:55:30,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:55:30,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:55:30,810 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:30,810 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:55:30,810 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:30,811 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:30,811 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:55:30,811 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:30,811 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:55:30,811 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402530810Disabling compacts and flushes for region at 1732402530810Disabling writes for close at 1732402530811 (+1 ms)Writing region close event to WAL at 1732402530811Closed at 1732402530811
2024-11-23T22:55:30,811 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/.initializing
2024-11-23T22:55:30,811 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/WALs/dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,814 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C35201%2C1732402530408, suffix=, logDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/WALs/dd96e8dc8369,35201,1732402530408, archiveDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/oldWALs, maxLogs=10
2024-11-23T22:55:30,814 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C35201%2C1732402530408.1732402530814
2024-11-23T22:55:30,818 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/WALs/dd96e8dc8369,35201,1732402530408/dd96e8dc8369%2C35201%2C1732402530408.1732402530814
2024-11-23T22:55:30,819 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38537:38537),(127.0.0.1/127.0.0.1:37645:37645)]
2024-11-23T22:55:30,819 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:55:30,820 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:30,820 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,820 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:55:30,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:55:30,826 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:30,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:30,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:55:30,827 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:55:30,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:30,828 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:30,828 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:55:30,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:55:30,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:30,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:30,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:55:30,830 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:55:30,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:30,831 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:30,831 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,832 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,832 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,833 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,833 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,834 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:55:30,835 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:55:30,837 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:55:30,837 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817494, jitterRate=0.039497897028923035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:55:30,838 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402530820Initializing all the Stores at 1732402530821 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402530821Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402530824 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402530824Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402530824Cleaning up temporary data from old regions at 1732402530833 (+9 ms)Region opened successfully at 1732402530837 (+4 ms)
2024-11-23T22:55:30,838 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:55:30,841 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2a9d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:55:30,842 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:55:30,842 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:55:30,842 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:55:30,842 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:55:30,842 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:55:30,843 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:55:30,843 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:55:30,844 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:55:30,845 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:55:30,853 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:55:30,853 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:55:30,854 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:55:30,863 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:55:30,864 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:55:30,865 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:55:30,874 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:55:30,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:55:30,885 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:55:30,887 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:55:30,895 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:55:30,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:55:30,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:55:30,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,906 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,35201,1732402530408, sessionid=0x10169e8c4010000, setting cluster-up flag (Was=false)
2024-11-23T22:55:30,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,958 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:55:30,960 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:30,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:30,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:31,011 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:55:31,012 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:31,013 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:55:31,014 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:55:31,015 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:55:31,015 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:55:31,015 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,35201,1732402530408
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:55:31,016 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,017 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:55:31,017 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,018 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:55:31,018 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402561019
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:55:31,019 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:55:31,019 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:55:31,019 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,020 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:55:31,020 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:55:31,020 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:55:31,020 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:55:31,020 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:55:31,024 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402531020,5,FailOnTimeoutGroup]
2024-11-23T22:55:31,030 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402531024,5,FailOnTimeoutGroup]
2024-11-23T22:55:31,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:55:31,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,030 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:55:31,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:55:31,034 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:55:31,034 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392
2024-11-23T22:55:31,039 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(746): ClusterId : 92c3a864-f3ac-44ea-a493-c92f62ff9095
2024-11-23T22:55:31,039 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:55:31,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:55:31,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:55:31,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:31,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:55:31,044 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:55:31,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:55:31,046 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:55:31,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,046 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,046 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:55:31,047 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:55:31,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,048 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,048 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:55:31,049 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:55:31,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,049 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:55:31,050 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:55:31,050 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:55:31,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740
2024-11-23T22:55:31,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740
2024-11-23T22:55:31,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:55:31,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:55:31,052 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:55:31,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:55:31,055 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:55:31,056 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752349, jitterRate=-0.04333996772766113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402531042Initializing all the Stores at 1732402531042Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531043 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531043Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402531043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531043Cleaning up temporary data from old regions at 1732402531052 (+9 ms)Region opened successfully at 1732402531056 (+4 ms)
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:55:31,057 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:55:31,057 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:55:31,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402531057Disabling compacts and flushes for region at 1732402531057Disabling writes for close at 1732402531057Writing region close event to WAL at 1732402531057Closed at 1732402531057
2024-11-23T22:55:31,058 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:55:31,058 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:55:31,058 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:55:31,060 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:55:31,060 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:55:31,060 DEBUG [RS:0;dd96e8dc8369:46461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f12dbc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:55:31,061 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:55:31,073 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:46461
2024-11-23T22:55:31,074 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:55:31,074 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:55:31,074 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:55:31,074 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,35201,1732402530408 with port=46461, startcode=1732402530608
2024-11-23T22:55:31,075 DEBUG [RS:0;dd96e8dc8369:46461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:55:31,077 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50391, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:55:31,077 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35201 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,077 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35201 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,079 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392
2024-11-23T22:55:31,079 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39749
2024-11-23T22:55:31,079 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:55:31,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:55:31,091 DEBUG [RS:0;dd96e8dc8369:46461 {}] zookeeper.ZKUtil(111): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,092 WARN  [RS:0;dd96e8dc8369:46461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:55:31,092 INFO  [RS:0;dd96e8dc8369:46461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:55:31,092 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,093 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,46461,1732402530608]
2024-11-23T22:55:31,102 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:55:31,104 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:55:31,104 INFO  [RS:0;dd96e8dc8369:46461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:55:31,104 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,104 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:55:31,105 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:55:31,105 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,105 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:55:31,106 DEBUG [RS:0;dd96e8dc8369:46461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:55:31,106 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,107 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,107 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,107 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,107 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,107 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46461,1732402530608-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:55:31,124 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:55:31,124 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,46461,1732402530608-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,125 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,125 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.Replication(171): dd96e8dc8369,46461,1732402530608 started
2024-11-23T22:55:31,143 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,143 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,46461,1732402530608, RpcServer on dd96e8dc8369/172.17.0.2:46461, sessionid=0x10169e8c4010001
2024-11-23T22:55:31,143 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:55:31,143 DEBUG [RS:0;dd96e8dc8369:46461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,143 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,46461,1732402530608'
2024-11-23T22:55:31,143 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,46461,1732402530608'
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:55:31,144 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:55:31,145 DEBUG [RS:0;dd96e8dc8369:46461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:55:31,145 INFO  [RS:0;dd96e8dc8369:46461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:55:31,145 INFO  [RS:0;dd96e8dc8369:46461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:55:31,210 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:31,211 WARN  [dd96e8dc8369:35201 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:55:31,247 INFO  [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C46461%2C1732402530608, suffix=, logDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608, archiveDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs, maxLogs=32
2024-11-23T22:55:31,247 INFO  [RS:0;dd96e8dc8369:46461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46461%2C1732402530608.1732402531247
2024-11-23T22:55:31,252 INFO  [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402531247
2024-11-23T22:55:31,253 DEBUG [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38537:38537),(127.0.0.1/127.0.0.1:37645:37645)]
2024-11-23T22:55:31,461 DEBUG [dd96e8dc8369:35201 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:55:31,462 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,463 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,46461,1732402530608, state=OPENING
2024-11-23T22:55:31,474 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:55:31,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:31,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:31,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:55:31,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:55:31,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:55:31,486 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:55:31,486 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,46461,1732402530608}]
2024-11-23T22:55:31,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:31,648 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:55:31,653 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:55:31,657 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:55:31,657 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:55:31,659 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C46461%2C1732402530608.meta, suffix=.meta, logDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608, archiveDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs, maxLogs=32
2024-11-23T22:55:31,659 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46461%2C1732402530608.meta.1732402531659.meta
2024-11-23T22:55:31,664 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.meta.1732402531659.meta
2024-11-23T22:55:31,665 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37645:37645),(127.0.0.1/127.0.0.1:38537:38537)]
2024-11-23T22:55:31,666 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:55:31,666 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:55:31,666 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:55:31,666 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:55:31,666 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:55:31,667 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:31,667 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:55:31,667 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:55:31,668 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:55:31,669 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:55:31,669 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,669 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,669 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:55:31,670 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:55:31,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,671 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,671 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:55:31,671 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:55:31,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,672 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,672 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:55:31,673 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:55:31,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,673 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:55:31,673 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:55:31,674 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740
2024-11-23T22:55:31,675 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740
2024-11-23T22:55:31,676 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:55:31,676 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:55:31,677 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:55:31,678 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:55:31,679 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=843704, jitterRate=0.07282605767250061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:55:31,679 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:55:31,680 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402531667Writing region info on filesystem at 1732402531667Initializing all the Stores at 1732402531667Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531668 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531668Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402531668Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402531668Cleaning up temporary data from old regions at 1732402531676 (+8 ms)Running coprocessor post-open hooks at 1732402531679 (+3 ms)Region opened successfully at 1732402531680 (+1 ms)
2024-11-23T22:55:31,681 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402531648
2024-11-23T22:55:31,684 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:55:31,684 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:55:31,685 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,686 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,46461,1732402530608, state=OPEN
2024-11-23T22:55:31,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:55:31,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:55:31,721 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:55:31,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:55:31,724 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:55:31,725 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,46461,1732402530608 in 235 msec
2024-11-23T22:55:31,727 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:55:31,727 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 667 msec
2024-11-23T22:55:31,728 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:55:31,728 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:55:31,730 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:55:31,730 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,46461,1732402530608, seqNum=-1]
2024-11-23T22:55:31,730 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:55:31,731 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44351, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:55:31,737 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 722 msec
2024-11-23T22:55:31,737 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402531737, completionTime=-1
2024-11-23T22:55:31,737 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:55:31,737 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402591740
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402651740
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:35201, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,740 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,741 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,742 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.007sec
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:55:31,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:55:31,747 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:55:31,747 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:55:31,747 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35201,1732402530408-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:55:31,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876b14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:55:31,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,35201,-1 for getting cluster id
2024-11-23T22:55:31,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:55:31,750 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '92c3a864-f3ac-44ea-a493-c92f62ff9095'
2024-11-23T22:55:31,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:55:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "92c3a864-f3ac-44ea-a493-c92f62ff9095"

2024-11-23T22:55:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab76b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:55:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,35201,-1]
2024-11-23T22:55:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:55:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:55:31,752 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:55:31,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a479c4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:55:31,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:55:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,46461,1732402530608, seqNum=-1]
2024-11-23T22:55:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:55:31,755 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:55:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:31,757 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:55:31,759 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:55:31,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-23T22:55:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,35201,1732402530408
2024-11-23T22:55:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@31acdfc7
2024-11-23T22:55:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-23T22:55:31,762 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-23T22:55:31,763 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-23T22:55:31,763 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-23T22:55:31,763 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:55:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-23T22:55:31,765 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-23T22:55:31,766 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:31,766 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-23T22:55:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:55:31,767 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-23T22:55:31,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741835_1011 (size=381)
2024-11-23T22:55:31,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741835_1011 (size=381)
2024-11-23T22:55:31,774 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a51d8942b5b3c2e061b44a1459cc889, NAME => 'TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392
2024-11-23T22:55:31,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741836_1012 (size=64)
2024-11-23T22:55:31,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741836_1012 (size=64)
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 0a51d8942b5b3c2e061b44a1459cc889, disabling compactions & flushes
2024-11-23T22:55:31,781 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. after waiting 0 ms
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:31,781 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:31,781 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a51d8942b5b3c2e061b44a1459cc889:
Waiting for close lock at 1732402531781Disabling compacts and flushes for region at 1732402531781Disabling writes for close at 1732402531781Writing region close event to WAL at 1732402531781Closed at 1732402531781
2024-11-23T22:55:31,782 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-23T22:55:31,783 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732402531782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402531782"}]},"ts":"1732402531782"}
2024-11-23T22:55:31,785 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-23T22:55:31,786 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-23T22:55:31,786 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402531786"}]},"ts":"1732402531786"}
2024-11-23T22:55:31,788 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-23T22:55:31,789 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, ASSIGN}]
2024-11-23T22:55:31,790 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, ASSIGN
2024-11-23T22:55:31,791 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, ASSIGN; state=OFFLINE, location=dd96e8dc8369,46461,1732402530608; forceNewPlan=false, retain=false
2024-11-23T22:55:31,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-23T22:55:31,942 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0a51d8942b5b3c2e061b44a1459cc889, regionState=OPENING, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:31,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, ASSIGN because future has completed
2024-11-23T22:55:31,945 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608}]
2024-11-23T22:55:32,101 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:32,101 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a51d8942b5b3c2e061b44a1459cc889, NAME => 'TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:55:32,101 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,101 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:32,101 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,101 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,107 INFO  [StoreOpener-0a51d8942b5b3c2e061b44a1459cc889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0a51d8942b5b3c2e061b44a1459cc889 
2024-11-23T22:55:32,108 INFO  [StoreOpener-0a51d8942b5b3c2e061b44a1459cc889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a51d8942b5b3c2e061b44a1459cc889 columnFamilyName info
2024-11-23T22:55:32,108 DEBUG [StoreOpener-0a51d8942b5b3c2e061b44a1459cc889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:32,109 INFO  [StoreOpener-0a51d8942b5b3c2e061b44a1459cc889-1 {}] regionserver.HStore(327): Store=0a51d8942b5b3c2e061b44a1459cc889/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:32,109 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,109 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,110 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,110 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,110 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,112 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,114 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:55:32,114 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0a51d8942b5b3c2e061b44a1459cc889; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802080, jitterRate=0.019897952675819397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:55:32,114 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:32,115 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0a51d8942b5b3c2e061b44a1459cc889:
Running coprocessor pre-open hook at 1732402532102Writing region info on filesystem at 1732402532102Initializing all the Stores at 1732402532102Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402532102Cleaning up temporary data from old regions at 1732402532110 (+8 ms)Running coprocessor post-open hooks at 1732402532114 (+4 ms)Region opened successfully at 1732402532115 (+1 ms)
2024-11-23T22:55:32,116 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., pid=6, masterSystemTime=1732402532097
2024-11-23T22:55:32,118 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:32,118 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:32,119 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0a51d8942b5b3c2e061b44a1459cc889, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:32,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608 because future has completed
2024-11-23T22:55:32,124 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-23T22:55:32,125 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608 in 177 msec
2024-11-23T22:55:32,127 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-23T22:55:32,127 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, ASSIGN in 335 msec
2024-11-23T22:55:32,128 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-23T22:55:32,128 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402532128"}]},"ts":"1732402532128"}
2024-11-23T22:55:32,130 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-23T22:55:32,131 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-23T22:55:32,133 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 368 msec
2024-11-23T22:55:32,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,202 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,202 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:32,223 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,223 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,223 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,223 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,227 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,227 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:32,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:32,734 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:55:32,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,766 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:32,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:33,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:33,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:33,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:34,212 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:34,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:34,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:35,213 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:35,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:35,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:36,213 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:36,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:36,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:37,214 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:37,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:37,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:37,604 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:55:37,605 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,605 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,606 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,606 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,606 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,607 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,607 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,608 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,634 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,634 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,635 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,635 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,635 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,635 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,639 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,642 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:37,649 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:55:37,649 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-23T22:55:38,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:38,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:38,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:39,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:39,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:39,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:40,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:40,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:40,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:41,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:41,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:41,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35201 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-23T22:55:41,806 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-23T22:55:41,806 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-23T22:55:41,810 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-23T22:55:41,810 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., hostname=dd96e8dc8369,46461,1732402530608, seqNum=2]
2024-11-23T22:55:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:41,828 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:55:41,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1f15f50a752f46388664cbf9dca67053 is 1080, key is row0001/info:/1732402541815/Put/seqid=0
2024-11-23T22:55:41,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741837_1013 (size=12509)
2024-11-23T22:55:41,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741837_1013 (size=12509)
2024-11-23T22:55:41,854 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1f15f50a752f46388664cbf9dca67053
2024-11-23T22:55:41,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1f15f50a752f46388664cbf9dca67053 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053
2024-11-23T22:55:41,869 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053, entries=7, sequenceid=11, filesize=12.2 K
2024-11-23T22:55:41,870 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for 0a51d8942b5b3c2e061b44a1459cc889 in 42ms, sequenceid=11, compaction requested=false
2024-11-23T22:55:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:41,872 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB
2024-11-23T22:55:41,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1cbc21bf5c444b75965092ca3a70c451 is 1080, key is row0008/info:/1732402541829/Put/seqid=0
2024-11-23T22:55:41,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741838_1014 (size=26530)
2024-11-23T22:55:41,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741838_1014 (size=26530)
2024-11-23T22:55:41,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-23T22:55:41,913 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-23T22:55:42,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:42,282 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1cbc21bf5c444b75965092ca3a70c451
2024-11-23T22:55:42,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/1cbc21bf5c444b75965092ca3a70c451 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451
2024-11-23T22:55:42,299 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451, entries=20, sequenceid=34, filesize=25.9 K
2024-11-23T22:55:42,302 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for 0a51d8942b5b3c2e061b44a1459cc889 in 431ms, sequenceid=34, compaction requested=false
2024-11-23T22:55:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K
2024-11-23T22:55:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451 because midkey is the same as first or last row
2024-11-23T22:55:42,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:42,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:43,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:43,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:43,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:43,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:55:43,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/39a47e086af948beb949a21b7dc42b83 is 1080, key is row0028/info:/1732402541873/Put/seqid=0
2024-11-23T22:55:43,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741839_1015 (size=12509)
2024-11-23T22:55:43,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741839_1015 (size=12509)
2024-11-23T22:55:43,912 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/39a47e086af948beb949a21b7dc42b83
2024-11-23T22:55:43,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/39a47e086af948beb949a21b7dc42b83 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83
2024-11-23T22:55:43,929 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83, entries=7, sequenceid=44, filesize=12.2 K
2024-11-23T22:55:43,930 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 0a51d8942b5b3c2e061b44a1459cc889 in 42ms, sequenceid=44, compaction requested=true
2024-11-23T22:55:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451 because midkey is the same as first or last row
2024-11-23T22:55:43,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a51d8942b5b3c2e061b44a1459cc889:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:55:43,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:43,931 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:43,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-23T22:55:43,934 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:55:43,934 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): 0a51d8942b5b3c2e061b44a1459cc889/info is initiating minor compaction (all files)
2024-11-23T22:55:43,934 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a51d8942b5b3c2e061b44a1459cc889/info in TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:43,934 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp, totalSize=50.3 K
2024-11-23T22:55:43,935 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f15f50a752f46388664cbf9dca67053, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732402541815
2024-11-23T22:55:43,935 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cbc21bf5c444b75965092ca3a70c451, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732402541829
2024-11-23T22:55:43,937 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a47e086af948beb949a21b7dc42b83, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732402541873
2024-11-23T22:55:43,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/d43ecce77e0d446f8e5243ae8ccc683e is 1080, key is row0035/info:/1732402543891/Put/seqid=0
2024-11-23T22:55:43,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741840_1016 (size=18987)
2024-11-23T22:55:43,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741840_1016 (size=18987)
2024-11-23T22:55:43,958 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/d43ecce77e0d446f8e5243ae8ccc683e
2024-11-23T22:55:43,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/d43ecce77e0d446f8e5243ae8ccc683e as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e
2024-11-23T22:55:43,976 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a51d8942b5b3c2e061b44a1459cc889#info#compaction#61 average throughput is 17.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:43,976 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/cabf14d4f1214219b1bd9c885c3736f6 is 1080, key is row0001/info:/1732402541815/Put/seqid=0
2024-11-23T22:55:43,979 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e, entries=13, sequenceid=60, filesize=18.5 K
2024-11-23T22:55:43,980 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 0a51d8942b5b3c2e061b44a1459cc889 in 48ms, sequenceid=60, compaction requested=false
2024-11-23T22:55:43,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:43,980 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K
2024-11-23T22:55:43,980 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:43,980 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451 because midkey is the same as first or last row
2024-11-23T22:55:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:43,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-23T22:55:44,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/c46a94773dfd4aef8e299fb755ef3c62 is 1080, key is row0048/info:/1732402543934/Put/seqid=0
2024-11-23T22:55:44,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741841_1017 (size=41747)
2024-11-23T22:55:44,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741841_1017 (size=41747)
2024-11-23T22:55:44,009 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/cabf14d4f1214219b1bd9c885c3736f6 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6
2024-11-23T22:55:44,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741842_1018 (size=18987)
2024-11-23T22:55:44,018 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a51d8942b5b3c2e061b44a1459cc889/info of 0a51d8942b5b3c2e061b44a1459cc889 into cabf14d4f1214219b1bd9c885c3736f6(size=40.8 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:44,018 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:44,018 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., storeName=0a51d8942b5b3c2e061b44a1459cc889/info, priority=13, startTime=1732402543930; duration=0sec
2024-11-23T22:55:44,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741842_1018 (size=18987)
2024-11-23T22:55:44,018 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,018 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6 because midkey is the same as first or last row
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6 because midkey is the same as first or last row
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6 because midkey is the same as first or last row
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:44,019 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a51d8942b5b3c2e061b44a1459cc889:info
2024-11-23T22:55:44,020 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/c46a94773dfd4aef8e299fb755ef3c62
2024-11-23T22:55:44,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/c46a94773dfd4aef8e299fb755ef3c62 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62
2024-11-23T22:55:44,033 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62, entries=13, sequenceid=76, filesize=18.5 K
2024-11-23T22:55:44,034 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for 0a51d8942b5b3c2e061b44a1459cc889 in 51ms, sequenceid=76, compaction requested=true
2024-11-23T22:55:44,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:44,035 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=77.9 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,035 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,035 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6 because midkey is the same as first or last row
2024-11-23T22:55:44,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a51d8942b5b3c2e061b44a1459cc889:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:55:44,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:44,035 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:44,036 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 79721 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:55:44,036 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1541): 0a51d8942b5b3c2e061b44a1459cc889/info is initiating minor compaction (all files)
2024-11-23T22:55:44,036 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a51d8942b5b3c2e061b44a1459cc889/info in TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:44,036 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp, totalSize=77.9 K
2024-11-23T22:55:44,037 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting cabf14d4f1214219b1bd9c885c3736f6, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732402541815
2024-11-23T22:55:44,037 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting d43ecce77e0d446f8e5243ae8ccc683e, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732402543891
2024-11-23T22:55:44,038 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting c46a94773dfd4aef8e299fb755ef3c62, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732402543934
2024-11-23T22:55:44,052 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a51d8942b5b3c2e061b44a1459cc889#info#compaction#63 average throughput is 20.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:44,053 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/0904c4d3774846a0853f935013ef7617 is 1080, key is row0001/info:/1732402541815/Put/seqid=0
2024-11-23T22:55:44,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741843_1019 (size=69920)
2024-11-23T22:55:44,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741843_1019 (size=69920)
2024-11-23T22:55:44,070 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/0904c4d3774846a0853f935013ef7617 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617
2024-11-23T22:55:44,079 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a51d8942b5b3c2e061b44a1459cc889/info of 0a51d8942b5b3c2e061b44a1459cc889 into 0904c4d3774846a0853f935013ef7617(size=68.3 K), total size for store is 68.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:44,079 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., storeName=0a51d8942b5b3c2e061b44a1459cc889/info, priority=13, startTime=1732402544035; duration=0sec
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 because midkey is the same as first or last row
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 because midkey is the same as first or last row
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 because midkey is the same as first or last row
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:44,079 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a51d8942b5b3c2e061b44a1459cc889:info
2024-11-23T22:55:44,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:44,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:44,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:45,219 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:45,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:45,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,000 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:55:46,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/3955af476e57432aa4e86a87da39e67d is 1080, key is row0061/info:/1732402543984/Put/seqid=0
2024-11-23T22:55:46,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741844_1020 (size=12509)
2024-11-23T22:55:46,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741844_1020 (size=12509)
2024-11-23T22:55:46,011 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/3955af476e57432aa4e86a87da39e67d
2024-11-23T22:55:46,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/3955af476e57432aa4e86a87da39e67d as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d
2024-11-23T22:55:46,024 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d, entries=7, sequenceid=88, filesize=12.2 K
2024-11-23T22:55:46,025 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 0a51d8942b5b3c2e061b44a1459cc889 in 25ms, sequenceid=88, compaction requested=false
2024-11-23T22:55:46,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:46,025 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=80.5 K, sizeToCheck=16.0 K
2024-11-23T22:55:46,025 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:46,025 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 because midkey is the same as first or last row
2024-11-23T22:55:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,027 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-23T22:55:46,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/b7ba6c1065eb40db9e3538edc3bf2e1e is 1080, key is row0068/info:/1732402546002/Put/seqid=0
2024-11-23T22:55:46,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741845_1021 (size=16817)
2024-11-23T22:55:46,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741845_1021 (size=16817)
2024-11-23T22:55:46,037 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/b7ba6c1065eb40db9e3538edc3bf2e1e
2024-11-23T22:55:46,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/b7ba6c1065eb40db9e3538edc3bf2e1e as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e
2024-11-23T22:55:46,051 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e, entries=11, sequenceid=102, filesize=16.4 K
2024-11-23T22:55:46,052 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 0a51d8942b5b3c2e061b44a1459cc889 in 25ms, sequenceid=102, compaction requested=true
2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.9 K, sizeToCheck=16.0 K
2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 because midkey is the same as first or last row
2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0a51d8942b5b3c2e061b44a1459cc889:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:55:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:46,052 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,053 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-23T22:55:46,053 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 99246 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:55:46,053 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): 0a51d8942b5b3c2e061b44a1459cc889/info is initiating minor compaction (all files)
2024-11-23T22:55:46,053 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0a51d8942b5b3c2e061b44a1459cc889/info in TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,054 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp, totalSize=96.9 K
2024-11-23T22:55:46,054 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0904c4d3774846a0853f935013ef7617, keycount=60, bloomtype=ROW, size=68.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732402541815
2024-11-23T22:55:46,054 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3955af476e57432aa4e86a87da39e67d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732402543984
2024-11-23T22:55:46,055 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7ba6c1065eb40db9e3538edc3bf2e1e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732402546002
2024-11-23T22:55:46,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/7cda7c9671dc421db7c541de75cc893e is 1080, key is row0079/info:/1732402546028/Put/seqid=0
2024-11-23T22:55:46,063 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741846_1022 (size=17894)
2024-11-23T22:55:46,063 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741846_1022 (size=17894)
2024-11-23T22:55:46,070 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0a51d8942b5b3c2e061b44a1459cc889#info#compaction#67 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:46,071 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/4c1dd610c4c54bb4bf368c95871becf4 is 1080, key is row0001/info:/1732402541815/Put/seqid=0
2024-11-23T22:55:46,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741847_1023 (size=89485)
2024-11-23T22:55:46,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741847_1023 (size=89485)
2024-11-23T22:55:46,081 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/4c1dd610c4c54bb4bf368c95871becf4 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4
2024-11-23T22:55:46,088 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0a51d8942b5b3c2e061b44a1459cc889/info of 0a51d8942b5b3c2e061b44a1459cc889 into 4c1dd610c4c54bb4bf368c95871becf4(size=87.4 K), total size for store is 87.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:46,088 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:46,088 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., storeName=0a51d8942b5b3c2e061b44a1459cc889/info, priority=13, startTime=1732402546052; duration=0sec
2024-11-23T22:55:46,088 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=87.4 K, sizeToCheck=16.0 K
2024-11-23T22:55:46,088 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:46,088 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=87.4 K, sizeToCheck=16.0 K
2024-11-23T22:55:46,088 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:46,089 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=87.4 K, sizeToCheck=16.0 K
2024-11-23T22:55:46,089 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-23T22:55:46,090 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:46,090 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:46,090 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0a51d8942b5b3c2e061b44a1459cc889:info
2024-11-23T22:55:46,091 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35201 {}] assignment.AssignmentManager(1363): Split request from dd96e8dc8369,46461,1732402530608, parent={ENCODED => 0a51d8942b5b3c2e061b44a1459cc889, NAME => 'TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-23T22:55:46,096 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35201 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:46,100 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35201 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=0a51d8942b5b3c2e061b44a1459cc889, daughterA=fbbb15916746a6267d266e9a976a42a3, daughterB=c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,101 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=0a51d8942b5b3c2e061b44a1459cc889, daughterA=fbbb15916746a6267d266e9a976a42a3, daughterB=c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,101 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=0a51d8942b5b3c2e061b44a1459cc889, daughterA=fbbb15916746a6267d266e9a976a42a3, daughterB=c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,101 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=0a51d8942b5b3c2e061b44a1459cc889, daughterA=fbbb15916746a6267d266e9a976a42a3, daughterB=c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,108 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, UNASSIGN}]
2024-11-23T22:55:46,110 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, UNASSIGN
2024-11-23T22:55:46,111 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a51d8942b5b3c2e061b44a1459cc889, regionState=CLOSING, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:46,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, UNASSIGN because future has completed
2024-11-23T22:55:46,114 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-23T22:55:46,114 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608}]
2024-11-23T22:55:46,220 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:46,271 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,271 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-23T22:55:46,272 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 0a51d8942b5b3c2e061b44a1459cc889, disabling compactions & flushes
2024-11-23T22:55:46,272 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,464 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/7cda7c9671dc421db7c541de75cc893e
2024-11-23T22:55:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/7cda7c9671dc421db7c541de75cc893e as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/7cda7c9671dc421db7c541de75cc893e
2024-11-23T22:55:46,480 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/7cda7c9671dc421db7c541de75cc893e, entries=12, sequenceid=117, filesize=17.5 K
2024-11-23T22:55:46,481 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=6.30 KB/6456 for 0a51d8942b5b3c2e061b44a1459cc889 in 428ms, sequenceid=117, compaction requested=false
2024-11-23T22:55:46,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0a51d8942b5b3c2e061b44a1459cc889:

2024-11-23T22:55:46,481 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,481 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,481 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. after waiting 0 ms
2024-11-23T22:55:46,481 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,482 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 0a51d8942b5b3c2e061b44a1459cc889 1/1 column families, dataSize=6.30 KB heapSize=7 KB
2024-11-23T22:55:46,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:46,486 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/edb018425fdf4abd97aa4366c9be5854 is 1080, key is row0091/info:/1732402546054/Put/seqid=0
2024-11-23T22:55:46,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741848_1024 (size=11424)
2024-11-23T22:55:46,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741848_1024 (size=11424)
2024-11-23T22:55:46,491 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/edb018425fdf4abd97aa4366c9be5854
2024-11-23T22:55:46,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:46,497 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/.tmp/info/edb018425fdf4abd97aa4366c9be5854 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/edb018425fdf4abd97aa4366c9be5854
2024-11-23T22:55:46,502 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/edb018425fdf4abd97aa4366c9be5854, entries=6, sequenceid=127, filesize=11.2 K
2024-11-23T22:55:46,503 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 0a51d8942b5b3c2e061b44a1459cc889 in 22ms, sequenceid=127, compaction requested=true
2024-11-23T22:55:46,504 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e] to archive
2024-11-23T22:55:46,505 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:55:46,507 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1f15f50a752f46388664cbf9dca67053
2024-11-23T22:55:46,508 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/1cbc21bf5c444b75965092ca3a70c451
2024-11-23T22:55:46,509 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/cabf14d4f1214219b1bd9c885c3736f6
2024-11-23T22:55:46,510 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/39a47e086af948beb949a21b7dc42b83
2024-11-23T22:55:46,511 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/d43ecce77e0d446f8e5243ae8ccc683e
2024-11-23T22:55:46,512 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/0904c4d3774846a0853f935013ef7617
2024-11-23T22:55:46,513 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/c46a94773dfd4aef8e299fb755ef3c62
2024-11-23T22:55:46,514 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/3955af476e57432aa4e86a87da39e67d
2024-11-23T22:55:46,516 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/b7ba6c1065eb40db9e3538edc3bf2e1e
2024-11-23T22:55:46,521 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-23T22:55:46,522 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.
2024-11-23T22:55:46,522 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 0a51d8942b5b3c2e061b44a1459cc889:
Waiting for close lock at 1732402546272Running coprocessor pre-close hooks at 1732402546272Disabling compacts and flushes for region at 1732402546272Disabling writes for close at 1732402546481 (+209 ms)Obtaining lock to block concurrent updates at 1732402546482 (+1 ms)Preparing flush snapshotting stores in 0a51d8942b5b3c2e061b44a1459cc889 at 1732402546482Finished memstore snapshotting TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., syncing WAL and waiting on mvcc, flushsize=dataSize=6456, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1732402546482Flushing stores of TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. at 1732402546482Flushing 0a51d8942b5b3c2e061b44a1459cc889/info: creating writer at 1732402546483 (+1 ms)Flushing 0a51d8942b5b3c2e061b44a1459cc889/info: appending metadata at 1732402546485 (+2 ms)Flushing 0a51d8942b5b3c2e061b44a1459cc889/info: closing flushed file at 1732402546485Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42799086: reopening flushed file at 1732402546496 (+11 ms)Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 0a51d8942b5b3c2e061b44a1459cc889 in 22ms, sequenceid=127, compaction requested=true at 1732402546503 (+7 ms)Writing region close event to WAL at 1732402546518 (+15 ms)Running coprocessor post-close hooks at 1732402546522 (+4 ms)Closed at 1732402546522
2024-11-23T22:55:46,524 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,525 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a51d8942b5b3c2e061b44a1459cc889, regionState=CLOSED
2024-11-23T22:55:46,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608 because future has completed
2024-11-23T22:55:46,530 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-23T22:55:46,530 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 0a51d8942b5b3c2e061b44a1459cc889, server=dd96e8dc8369,46461,1732402530608 in 414 msec
2024-11-23T22:55:46,533 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-23T22:55:46,533 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0a51d8942b5b3c2e061b44a1459cc889, UNASSIGN in 422 msec
2024-11-23T22:55:46,540 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:46,543 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=0a51d8942b5b3c2e061b44a1459cc889, threads=3
2024-11-23T22:55:46,546 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4 for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,546 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/7cda7c9671dc421db7c541de75cc893e for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,546 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/edb018425fdf4abd97aa4366c9be5854 for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,557 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/edb018425fdf4abd97aa4366c9be5854, top=true
2024-11-23T22:55:46,557 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/7cda7c9671dc421db7c541de75cc893e, top=true
2024-11-23T22:55:46,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741849_1025 (size=27)
2024-11-23T22:55:46,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741849_1025 (size=27)
2024-11-23T22:55:46,567 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854 for child: c26e66673b075581870eddb8a2594cc9, parent: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,568 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/edb018425fdf4abd97aa4366c9be5854 for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,568 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e for child: c26e66673b075581870eddb8a2594cc9, parent: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,568 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/7cda7c9671dc421db7c541de75cc893e for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741850_1026 (size=27)
2024-11-23T22:55:46,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741850_1026 (size=27)
2024-11-23T22:55:46,578 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4 for region: 0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:55:46,580 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 0a51d8942b5b3c2e061b44a1459cc889 Daughter A: [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889] storefiles, Daughter B: [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854] storefiles.
2024-11-23T22:55:46,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741851_1027 (size=71)
2024-11-23T22:55:46,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741851_1027 (size=71)
2024-11-23T22:55:46,591 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:46,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741852_1028 (size=71)
2024-11-23T22:55:46,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741852_1028 (size=71)
2024-11-23T22:55:46,605 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:46,614 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-23T22:55:46,616 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-23T22:55:46,619 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732402546618"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732402546618"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732402546618"}]},"ts":"1732402546618"}
2024-11-23T22:55:46,619 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732402546618"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402546618"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732402546618"}]},"ts":"1732402546618"}
2024-11-23T22:55:46,619 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732402546618"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402546618"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732402546618"}]},"ts":"1732402546618"}
2024-11-23T22:55:46,637 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbbb15916746a6267d266e9a976a42a3, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c26e66673b075581870eddb8a2594cc9, ASSIGN}]
2024-11-23T22:55:46,638 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c26e66673b075581870eddb8a2594cc9, ASSIGN
2024-11-23T22:55:46,638 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbbb15916746a6267d266e9a976a42a3, ASSIGN
2024-11-23T22:55:46,639 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c26e66673b075581870eddb8a2594cc9, ASSIGN; state=SPLITTING_NEW, location=dd96e8dc8369,46461,1732402530608; forceNewPlan=false, retain=false
2024-11-23T22:55:46,639 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbbb15916746a6267d266e9a976a42a3, ASSIGN; state=SPLITTING_NEW, location=dd96e8dc8369,46461,1732402530608; forceNewPlan=false, retain=false
2024-11-23T22:55:46,790 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fbbb15916746a6267d266e9a976a42a3, regionState=OPENING, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:46,790 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c26e66673b075581870eddb8a2594cc9, regionState=OPENING, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:46,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbbb15916746a6267d266e9a976a42a3, ASSIGN because future has completed
2024-11-23T22:55:46,793 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbbb15916746a6267d266e9a976a42a3, server=dd96e8dc8369,46461,1732402530608}]
2024-11-23T22:55:46,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c26e66673b075581870eddb8a2594cc9, ASSIGN because future has completed
2024-11-23T22:55:46,794 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c26e66673b075581870eddb8a2594cc9, server=dd96e8dc8369,46461,1732402530608}]
2024-11-23T22:55:46,949 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:46,949 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => c26e66673b075581870eddb8a2594cc9, NAME => 'TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-23T22:55:46,949 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,949 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:46,949 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,949 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,951 INFO  [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c26e66673b075581870eddb8a2594cc9 
2024-11-23T22:55:46,951 INFO  [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c26e66673b075581870eddb8a2594cc9 columnFamilyName info
2024-11-23T22:55:46,951 DEBUG [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:46,964 DEBUG [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-top
2024-11-23T22:55:46,969 DEBUG [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e
2024-11-23T22:55:46,973 DEBUG [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854
2024-11-23T22:55:46,973 INFO  [StoreOpener-c26e66673b075581870eddb8a2594cc9-1 {}] regionserver.HStore(327): Store=c26e66673b075581870eddb8a2594cc9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:46,973 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,974 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,975 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,976 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,976 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,978 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,979 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened c26e66673b075581870eddb8a2594cc9; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782854, jitterRate=-0.00455079972743988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for c26e66673b075581870eddb8a2594cc9:
Running coprocessor pre-open hook at 1732402546950Writing region info on filesystem at 1732402546950Initializing all the Stores at 1732402546950Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402546950Cleaning up temporary data from old regions at 1732402546976 (+26 ms)Running coprocessor post-open hooks at 1732402546979 (+3 ms)Region opened successfully at 1732402546979
2024-11-23T22:55:46,980 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., pid=13, masterSystemTime=1732402546945
2024-11-23T22:55:46,980 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:55:46,981 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:46,981 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:46,982 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:46,982 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:55:46,982 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:46,982 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-top, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=116.0 K
2024-11-23T22:55:46,983 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:46,983 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:46,983 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:55:46,983 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732402541815
2024-11-23T22:55:46,983 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => fbbb15916746a6267d266e9a976a42a3, NAME => 'TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-23T22:55:46,983 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:46,984 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:55:46,984 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:46,984 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:46,984 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732402546028
2024-11-23T22:55:46,984 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c26e66673b075581870eddb8a2594cc9, regionState=OPEN, openSeqNum=131, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:46,984 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732402546054
2024-11-23T22:55:46,986 INFO  [StoreOpener-fbbb15916746a6267d266e9a976a42a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fbbb15916746a6267d266e9a976a42a3 
2024-11-23T22:55:46,986 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-23T22:55:46,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-23T22:55:46,986 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-23T22:55:46,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c26e66673b075581870eddb8a2594cc9, server=dd96e8dc8369,46461,1732402530608 because future has completed
2024-11-23T22:55:46,987 INFO  [StoreOpener-fbbb15916746a6267d266e9a976a42a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbbb15916746a6267d266e9a976a42a3 columnFamilyName info
2024-11-23T22:55:46,987 DEBUG [StoreOpener-fbbb15916746a6267d266e9a976a42a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:55:46,993 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-23T22:55:46,993 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c26e66673b075581870eddb8a2594cc9, server=dd96e8dc8369,46461,1732402530608 in 194 msec
2024-11-23T22:55:46,996 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c26e66673b075581870eddb8a2594cc9, ASSIGN in 356 msec
2024-11-23T22:55:47,001 DEBUG [StoreOpener-fbbb15916746a6267d266e9a976a42a3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-bottom
2024-11-23T22:55:47,001 INFO  [StoreOpener-fbbb15916746a6267d266e9a976a42a3-1 {}] regionserver.HStore(327): Store=fbbb15916746a6267d266e9a976a42a3/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:55:47,001 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,002 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,003 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,004 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,004 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,005 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/d039d32e379f4ea1bc26ed7690cba089 is 193, key is TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9./info:regioninfo/1732402546984/Put/seqid=0
2024-11-23T22:55:47,009 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened fbbb15916746a6267d266e9a976a42a3; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736279, jitterRate=-0.06377306580543518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-23T22:55:47,009 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:55:47,009 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for fbbb15916746a6267d266e9a976a42a3:
Running coprocessor pre-open hook at 1732402546984Writing region info on filesystem at 1732402546984Initializing all the Stores at 1732402546985 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402546985Cleaning up temporary data from old regions at 1732402547004 (+19 ms)Running coprocessor post-open hooks at 1732402547009 (+5 ms)Region opened successfully at 1732402547009
2024-11-23T22:55:47,010 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3., pid=12, masterSystemTime=1732402546945
2024-11-23T22:55:47,010 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store fbbb15916746a6267d266e9a976a42a3:info, priority=-2147483648, current under compaction store size is 2
2024-11-23T22:55:47,010 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-23T22:55:47,010 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:47,011 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:55:47,011 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): fbbb15916746a6267d266e9a976a42a3/info is initiating minor compaction (all files)
2024-11-23T22:55:47,011 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbbb15916746a6267d266e9a976a42a3/info in TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:55:47,011 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-bottom] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/.tmp, totalSize=87.4 K
2024-11-23T22:55:47,012 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732402541815
2024-11-23T22:55:47,013 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:55:47,013 INFO  [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:55:47,014 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#70 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:47,014 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/20811710f8154cb4ac426abcc3473227 is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:55:47,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741853_1029 (size=9882)
2024-11-23T22:55:47,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741853_1029 (size=9882)
2024-11-23T22:55:47,017 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fbbb15916746a6267d266e9a976a42a3, regionState=OPEN, openSeqNum=131, regionLocation=dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:47,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbbb15916746a6267d266e9a976a42a3, server=dd96e8dc8369,46461,1732402530608 because future has completed
2024-11-23T22:55:47,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741854_1030 (size=42984)
2024-11-23T22:55:47,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741854_1030 (size=42984)
2024-11-23T22:55:47,028 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/20811710f8154cb4ac426abcc3473227 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/20811710f8154cb4ac426abcc3473227
2024-11-23T22:55:47,035 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-23T22:55:47,035 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fbbb15916746a6267d266e9a976a42a3, server=dd96e8dc8369,46461,1732402530608 in 235 msec
2024-11-23T22:55:47,038 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-23T22:55:47,038 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fbbb15916746a6267d266e9a976a42a3, ASSIGN in 398 msec
2024-11-23T22:55:47,040 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into 20811710f8154cb4ac426abcc3473227(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:47,040 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:55:47,040 INFO  [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402546980; duration=0sec
2024-11-23T22:55:47,040 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=0a51d8942b5b3c2e061b44a1459cc889, daughterA=fbbb15916746a6267d266e9a976a42a3, daughterB=c26e66673b075581870eddb8a2594cc9 in 942 msec
2024-11-23T22:55:47,040 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:47,040 DEBUG [RS:0;dd96e8dc8369:46461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:55:47,045 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbbb15916746a6267d266e9a976a42a3#info#compaction#71 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:47,045 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/.tmp/info/160b679f76fb4fdebecb776c67bbb4f0 is 1080, key is row0001/info:/1732402541815/Put/seqid=0
2024-11-23T22:55:47,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741855_1031 (size=70862)
2024-11-23T22:55:47,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741855_1031 (size=70862)
2024-11-23T22:55:47,055 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/.tmp/info/160b679f76fb4fdebecb776c67bbb4f0 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/160b679f76fb4fdebecb776c67bbb4f0
2024-11-23T22:55:47,062 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in fbbb15916746a6267d266e9a976a42a3/info of fbbb15916746a6267d266e9a976a42a3 into 160b679f76fb4fdebecb776c67bbb4f0(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:47,062 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbbb15916746a6267d266e9a976a42a3:

2024-11-23T22:55:47,062 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3., storeName=fbbb15916746a6267d266e9a976a42a3/info, priority=15, startTime=1732402547010; duration=0sec
2024-11-23T22:55:47,062 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:47,062 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbbb15916746a6267d266e9a976a42a3:info
2024-11-23T22:55:47,221 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:47,421 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/d039d32e379f4ea1bc26ed7690cba089
2024-11-23T22:55:47,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/ns/b8cffb1be2e14f0bbb47361f89580806 is 43, key is default/ns:d/1732402531732/Put/seqid=0
2024-11-23T22:55:47,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741856_1032 (size=5153)
2024-11-23T22:55:47,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741856_1032 (size=5153)
2024-11-23T22:55:47,450 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/ns/b8cffb1be2e14f0bbb47361f89580806
2024-11-23T22:55:47,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/table/1f0f587c4b7641e38cb7d6183de4861d is 65, key is TestLogRolling-testLogRolling/table:state/1732402532128/Put/seqid=0
2024-11-23T22:55:47,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741857_1033 (size=5340)
2024-11-23T22:55:47,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741857_1033 (size=5340)
2024-11-23T22:55:47,476 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/table/1f0f587c4b7641e38cb7d6183de4861d
2024-11-23T22:55:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/d039d32e379f4ea1bc26ed7690cba089 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/info/d039d32e379f4ea1bc26ed7690cba089
2024-11-23T22:55:47,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:47,488 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/info/d039d32e379f4ea1bc26ed7690cba089, entries=30, sequenceid=17, filesize=9.7 K
2024-11-23T22:55:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/ns/b8cffb1be2e14f0bbb47361f89580806 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/ns/b8cffb1be2e14f0bbb47361f89580806
2024-11-23T22:55:47,495 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/ns/b8cffb1be2e14f0bbb47361f89580806, entries=2, sequenceid=17, filesize=5.0 K
2024-11-23T22:55:47,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/table/1f0f587c4b7641e38cb7d6183de4861d as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/table/1f0f587c4b7641e38cb7d6183de4861d
2024-11-23T22:55:47,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:47,501 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/table/1f0f587c4b7641e38cb7d6183de4861d, entries=2, sequenceid=17, filesize=5.2 K
2024-11-23T22:55:47,502 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 516ms, sequenceid=17, compaction requested=false
2024-11-23T22:55:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-23T22:55:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:39768 deadline: 1732402558069, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. is not online on dd96e8dc8369,46461,1732402530608
2024-11-23T22:55:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., hostname=dd96e8dc8369,46461,1732402530608, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., hostname=dd96e8dc8369,46461,1732402530608, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. is not online on dd96e8dc8369,46461,1732402530608
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-23T22:55:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., hostname=dd96e8dc8369,46461,1732402530608, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889. is not online on dd96e8dc8369,46461,1732402530608
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-23T22:55:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732402531762.0a51d8942b5b3c2e061b44a1459cc889., hostname=dd96e8dc8369,46461,1732402530608, seqNum=2 from cache
2024-11-23T22:55:48,222 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:48,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:48,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:49,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:49,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:49,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:50,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:50,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:50,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:51,224 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:51,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:51,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:51,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,552 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,552 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,552 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,552 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,557 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:51,559 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,066 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:55:52,067 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,067 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,069 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,070 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,070 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,094 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,095 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,095 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,095 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,095 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,095 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,101 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:55:52,225 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:52,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:52,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:53,225 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:53,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:53,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:54,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:54,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:54,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:55,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:55,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:55,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:56,227 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:56,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:56,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:57,228 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:57,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:57,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., hostname=dd96e8dc8369,46461,1732402530608, seqNum=131]
2024-11-23T22:55:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:58,145 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:55:58,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/27f92ca0aa1142a7a7bd3ce1366aec8b is 1080, key is row0097/info:/1732402558134/Put/seqid=0
2024-11-23T22:55:58,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741858_1034 (size=12516)
2024-11-23T22:55:58,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741858_1034 (size=12516)
2024-11-23T22:55:58,156 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/27f92ca0aa1142a7a7bd3ce1366aec8b
2024-11-23T22:55:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/27f92ca0aa1142a7a7bd3ce1366aec8b as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b
2024-11-23T22:55:58,169 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b, entries=7, sequenceid=141, filesize=12.2 K
2024-11-23T22:55:58,170 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for c26e66673b075581870eddb8a2594cc9 in 25ms, sequenceid=141, compaction requested=false
2024-11-23T22:55:58,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:55:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:58,172 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-23T22:55:58,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7a6e165240074fc2bd8a28b01edb0e16 is 1080, key is row0104/info:/1732402558146/Put/seqid=0
2024-11-23T22:55:58,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741859_1035 (size=19000)
2024-11-23T22:55:58,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741859_1035 (size=19000)
2024-11-23T22:55:58,181 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7a6e165240074fc2bd8a28b01edb0e16
2024-11-23T22:55:58,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7a6e165240074fc2bd8a28b01edb0e16 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16
2024-11-23T22:55:58,192 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16, entries=13, sequenceid=157, filesize=18.6 K
2024-11-23T22:55:58,193 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for c26e66673b075581870eddb8a2594cc9 in 21ms, sequenceid=157, compaction requested=true
2024-11-23T22:55:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:55:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:55:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:58,193 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:55:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:55:58,193 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-23T22:55:58,194 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:55:58,194 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:55:58,194 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:55:58,195 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/20811710f8154cb4ac426abcc3473227, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=72.8 K
2024-11-23T22:55:58,195 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20811710f8154cb4ac426abcc3473227, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732402543986
2024-11-23T22:55:58,195 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f92ca0aa1142a7a7bd3ce1366aec8b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732402558134
2024-11-23T22:55:58,196 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6e165240074fc2bd8a28b01edb0e16, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732402558146
2024-11-23T22:55:58,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4cf69d5cd0c548609da623535c4d648b is 1080, key is row0117/info:/1732402558173/Put/seqid=0
2024-11-23T22:55:58,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741860_1036 (size=16828)
2024-11-23T22:55:58,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741860_1036 (size=16828)
2024-11-23T22:55:58,215 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#77 average throughput is 28.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:55:58,216 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7d85aa83298345f68391896d5f5d015e is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:55:58,217 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4cf69d5cd0c548609da623535c4d648b
2024-11-23T22:55:58,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741861_1037 (size=64714)
2024-11-23T22:55:58,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741861_1037 (size=64714)
2024-11-23T22:55:58,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4cf69d5cd0c548609da623535c4d648b as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b
2024-11-23T22:55:58,227 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7d85aa83298345f68391896d5f5d015e as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7d85aa83298345f68391896d5f5d015e
2024-11-23T22:55:58,228 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b, entries=11, sequenceid=171, filesize=16.4 K
2024-11-23T22:55:58,228 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:58,229 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=1.05 KB/1076 for c26e66673b075581870eddb8a2594cc9 in 36ms, sequenceid=171, compaction requested=false
2024-11-23T22:55:58,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:55:58,233 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into 7d85aa83298345f68391896d5f5d015e(size=63.2 K), total size for store is 79.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:55:58,233 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:55:58,234 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402558193; duration=0sec
2024-11-23T22:55:58,234 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:55:58,234 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:55:58,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:58,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:59,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:59,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:55:59,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:00,208 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:56:00,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/5c9f1e06d6e14998a44837f8838a7017 is 1080, key is row0128/info:/1732402558195/Put/seqid=0
2024-11-23T22:56:00,218 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741862_1038 (size=12516)
2024-11-23T22:56:00,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741862_1038 (size=12516)
2024-11-23T22:56:00,219 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/5c9f1e06d6e14998a44837f8838a7017
2024-11-23T22:56:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/5c9f1e06d6e14998a44837f8838a7017 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017
2024-11-23T22:56:00,230 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:00,234 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017, entries=7, sequenceid=182, filesize=12.2 K
2024-11-23T22:56:00,235 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for c26e66673b075581870eddb8a2594cc9 in 26ms, sequenceid=182, compaction requested=true
2024-11-23T22:56:00,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:00,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:00,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:00,235 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:00,236 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-23T22:56:00,236 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 94058 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:00,236 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:00,236 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:00,236 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7d85aa83298345f68391896d5f5d015e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=91.9 K
2024-11-23T22:56:00,237 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d85aa83298345f68391896d5f5d015e, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732402543986
2024-11-23T22:56:00,237 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cf69d5cd0c548609da623535c4d648b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732402558173
2024-11-23T22:56:00,237 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c9f1e06d6e14998a44837f8838a7017, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732402558195
2024-11-23T22:56:00,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/83bbf8384d81451b8b2e93088e334c0d is 1080, key is row0135/info:/1732402560210/Put/seqid=0
2024-11-23T22:56:00,251 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#80 average throughput is 37.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:00,252 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/e8450c0323f5426fb68028e516bd5743 is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:00,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741863_1039 (size=17906)
2024-11-23T22:56:00,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741863_1039 (size=17906)
2024-11-23T22:56:00,262 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/83bbf8384d81451b8b2e93088e334c0d
2024-11-23T22:56:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/83bbf8384d81451b8b2e93088e334c0d as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d
2024-11-23T22:56:00,274 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d, entries=12, sequenceid=197, filesize=17.5 K
2024-11-23T22:56:00,275 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=14.71 KB/15064 for c26e66673b075581870eddb8a2594cc9 in 39ms, sequenceid=197, compaction requested=false
2024-11-23T22:56:00,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:00,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741864_1040 (size=84293)
2024-11-23T22:56:00,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741864_1040 (size=84293)
2024-11-23T22:56:00,283 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/e8450c0323f5426fb68028e516bd5743 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e8450c0323f5426fb68028e516bd5743
2024-11-23T22:56:00,289 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into e8450c0323f5426fb68028e516bd5743(size=82.3 K), total size for store is 99.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:00,289 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:00,289 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402560235; duration=0sec
2024-11-23T22:56:00,290 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:00,290 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:00,387 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-23T22:56:00,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:00,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:01,230 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:01,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:01,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:02,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:02,271 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-23T22:56:02,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/78eaf389fffa407ba5f73a98888da8e2 is 1080, key is row0147/info:/1732402560237/Put/seqid=0
2024-11-23T22:56:02,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741865_1041 (size=21156)
2024-11-23T22:56:02,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741865_1041 (size=21156)
2024-11-23T22:56:02,280 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/78eaf389fffa407ba5f73a98888da8e2
2024-11-23T22:56:02,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/78eaf389fffa407ba5f73a98888da8e2 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2
2024-11-23T22:56:02,291 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2, entries=15, sequenceid=216, filesize=20.7 K
2024-11-23T22:56:02,292 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for c26e66673b075581870eddb8a2594cc9 in 22ms, sequenceid=216, compaction requested=true
2024-11-23T22:56:02,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:02,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:02,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:02,292 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:02,293 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-23T22:56:02,293 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123355 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:02,294 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:02,294 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:02,294 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e8450c0323f5426fb68028e516bd5743, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=120.5 K
2024-11-23T22:56:02,294 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8450c0323f5426fb68028e516bd5743, keycount=73, bloomtype=ROW, size=82.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732402543986
2024-11-23T22:56:02,295 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bbf8384d81451b8b2e93088e334c0d, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732402560210
2024-11-23T22:56:02,295 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78eaf389fffa407ba5f73a98888da8e2, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732402560237
2024-11-23T22:56:02,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/81fd70822fe14b40be0f1241d30fdb70 is 1080, key is row0162/info:/1732402562272/Put/seqid=0
2024-11-23T22:56:02,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741866_1042 (size=16828)
2024-11-23T22:56:02,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741866_1042 (size=16828)
2024-11-23T22:56:02,305 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/81fd70822fe14b40be0f1241d30fdb70
2024-11-23T22:56:02,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/81fd70822fe14b40be0f1241d30fdb70 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70
2024-11-23T22:56:02,313 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#83 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:02,314 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/9681708186dc492097da13251c36c0fc is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:02,319 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70, entries=11, sequenceid=230, filesize=16.4 K
2024-11-23T22:56:02,320 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for c26e66673b075581870eddb8a2594cc9 in 27ms, sequenceid=230, compaction requested=false
2024-11-23T22:56:02,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:02,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741867_1043 (size=113509)
2024-11-23T22:56:02,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741867_1043 (size=113509)
2024-11-23T22:56:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:02,323 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-23T22:56:02,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7c1c332e14bd4cd8ace639b243f8c456 is 1080, key is row0173/info:/1732402562294/Put/seqid=0
2024-11-23T22:56:02,331 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/9681708186dc492097da13251c36c0fc as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/9681708186dc492097da13251c36c0fc
2024-11-23T22:56:02,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741868_1044 (size=19000)
2024-11-23T22:56:02,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741868_1044 (size=19000)
2024-11-23T22:56:02,335 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7c1c332e14bd4cd8ace639b243f8c456
2024-11-23T22:56:02,340 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into 9681708186dc492097da13251c36c0fc(size=110.8 K), total size for store is 127.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:02,340 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:02,340 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402562292; duration=0sec
2024-11-23T22:56:02,340 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:02,340 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:02,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/7c1c332e14bd4cd8ace639b243f8c456 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456
2024-11-23T22:56:02,347 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456, entries=13, sequenceid=246, filesize=18.6 K
2024-11-23T22:56:02,348 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=7.36 KB/7532 for c26e66673b075581870eddb8a2594cc9 in 26ms, sequenceid=246, compaction requested=true
2024-11-23T22:56:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:02,348 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:02,350 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 149337 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:02,350 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:02,350 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:02,350 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/9681708186dc492097da13251c36c0fc, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=145.8 K
2024-11-23T22:56:02,350 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9681708186dc492097da13251c36c0fc, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732402543986
2024-11-23T22:56:02,351 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81fd70822fe14b40be0f1241d30fdb70, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732402562272
2024-11-23T22:56:02,351 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c1c332e14bd4cd8ace639b243f8c456, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732402562294
2024-11-23T22:56:02,365 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#85 average throughput is 42.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:02,366 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/dfdf85a03899402dbc6933a3dae4d647 is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:02,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741869_1045 (size=139688)
2024-11-23T22:56:02,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741869_1045 (size=139688)
2024-11-23T22:56:02,375 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/dfdf85a03899402dbc6933a3dae4d647 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/dfdf85a03899402dbc6933a3dae4d647
2024-11-23T22:56:02,382 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into dfdf85a03899402dbc6933a3dae4d647(size=136.4 K), total size for store is 136.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:02,382 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:02,382 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402562348; duration=0sec
2024-11-23T22:56:02,382 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:02,382 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:02,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:02,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:03,232 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:03,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:03,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:04,232 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:04,345 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-23T22:56:04,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/09cc36aa228043568c6e55f8b12ebabd is 1080, key is row0186/info:/1732402562324/Put/seqid=0
2024-11-23T22:56:04,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741870_1046 (size=13595)
2024-11-23T22:56:04,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741870_1046 (size=13595)
2024-11-23T22:56:04,366 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/09cc36aa228043568c6e55f8b12ebabd
2024-11-23T22:56:04,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/09cc36aa228043568c6e55f8b12ebabd as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd
2024-11-23T22:56:04,377 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd, entries=8, sequenceid=259, filesize=13.3 K
2024-11-23T22:56:04,378 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=15.76 KB/16140 for c26e66673b075581870eddb8a2594cc9 in 33ms, sequenceid=259, compaction requested=false
2024-11-23T22:56:04,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:04,379 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-23T22:56:04,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/bd6c85d2de674b6088f01b2e6c0bf783 is 1080, key is row0194/info:/1732402564347/Put/seqid=0
2024-11-23T22:56:04,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741871_1047 (size=22254)
2024-11-23T22:56:04,392 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/bd6c85d2de674b6088f01b2e6c0bf783
2024-11-23T22:56:04,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741871_1047 (size=22254)
2024-11-23T22:56:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/bd6c85d2de674b6088f01b2e6c0bf783 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783
2024-11-23T22:56:04,403 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783, entries=16, sequenceid=278, filesize=21.7 K
2024-11-23T22:56:04,403 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=12.61 KB/12912 for c26e66673b075581870eddb8a2594cc9 in 24ms, sequenceid=278, compaction requested=true
2024-11-23T22:56:04,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:04,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:04,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:04,404 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:04,404 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-23T22:56:04,405 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 175537 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:04,405 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:04,405 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:04,405 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/dfdf85a03899402dbc6933a3dae4d647, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=171.4 K
2024-11-23T22:56:04,406 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfdf85a03899402dbc6933a3dae4d647, keycount=124, bloomtype=ROW, size=136.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732402543986
2024-11-23T22:56:04,406 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09cc36aa228043568c6e55f8b12ebabd, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732402562324
2024-11-23T22:56:04,406 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd6c85d2de674b6088f01b2e6c0bf783, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732402564347
2024-11-23T22:56:04,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/08015e54b7804fe68d6e5ca5d9305dbc is 1080, key is row0210/info:/1732402564380/Put/seqid=0
2024-11-23T22:56:04,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741872_1048 (size=19013)
2024-11-23T22:56:04,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741872_1048 (size=19013)
2024-11-23T22:56:04,418 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/08015e54b7804fe68d6e5ca5d9305dbc
2024-11-23T22:56:04,421 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#89 average throughput is 37.97 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:04,421 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/d9a3e31391414e83831dde882004906f is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/08015e54b7804fe68d6e5ca5d9305dbc as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc
2024-11-23T22:56:04,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741873_1049 (size=165683)
2024-11-23T22:56:04,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741873_1049 (size=165683)
2024-11-23T22:56:04,429 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc, entries=13, sequenceid=294, filesize=18.6 K
2024-11-23T22:56:04,430 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=2.10 KB/2152 for c26e66673b075581870eddb8a2594cc9 in 26ms, sequenceid=294, compaction requested=false
2024-11-23T22:56:04,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:04,432 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/d9a3e31391414e83831dde882004906f as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/d9a3e31391414e83831dde882004906f
2024-11-23T22:56:04,438 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into d9a3e31391414e83831dde882004906f(size=161.8 K), total size for store is 180.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:04,438 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:04,438 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402564404; duration=0sec
2024-11-23T22:56:04,438 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:04,438 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:04,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:04,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:05,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:05,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:05,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:06,234 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:06,417 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-23T22:56:06,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/b418cd67fe604c08a8eb522e2b86c05d is 1080, key is row0223/info:/1732402564405/Put/seqid=0
2024-11-23T22:56:06,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741874_1050 (size=12523)
2024-11-23T22:56:06,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741874_1050 (size=12523)
2024-11-23T22:56:06,432 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/b418cd67fe604c08a8eb522e2b86c05d
2024-11-23T22:56:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/b418cd67fe604c08a8eb522e2b86c05d as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d
2024-11-23T22:56:06,443 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d, entries=7, sequenceid=305, filesize=12.2 K
2024-11-23T22:56:06,444 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for c26e66673b075581870eddb8a2594cc9 in 27ms, sequenceid=305, compaction requested=true
2024-11-23T22:56:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:06,444 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:06,445 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 197219 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:06,445 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:06,445 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:06,446 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/d9a3e31391414e83831dde882004906f, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=192.6 K
2024-11-23T22:56:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:06,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-23T22:56:06,446 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a3e31391414e83831dde882004906f, keycount=148, bloomtype=ROW, size=161.8 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732402543986
2024-11-23T22:56:06,447 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08015e54b7804fe68d6e5ca5d9305dbc, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732402564380
2024-11-23T22:56:06,447 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b418cd67fe604c08a8eb522e2b86c05d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732402564405
2024-11-23T22:56:06,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/73cec92a57354bc281200bd0a8a20540 is 1080, key is row0230/info:/1732402566418/Put/seqid=0
2024-11-23T22:56:06,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741875_1051 (size=20092)
2024-11-23T22:56:06,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741875_1051 (size=20092)
2024-11-23T22:56:06,460 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#92 average throughput is 57.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:06,460 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/73cec92a57354bc281200bd0a8a20540
2024-11-23T22:56:06,461 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/2026127519424fcbb17442f6c87c3730 is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:06,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741876_1052 (size=187373)
2024-11-23T22:56:06,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741876_1052 (size=187373)
2024-11-23T22:56:06,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/73cec92a57354bc281200bd0a8a20540 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540
2024-11-23T22:56:06,471 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/2026127519424fcbb17442f6c87c3730 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/2026127519424fcbb17442f6c87c3730
2024-11-23T22:56:06,471 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540, entries=14, sequenceid=322, filesize=19.6 K
2024-11-23T22:56:06,472 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for c26e66673b075581870eddb8a2594cc9 in 26ms, sequenceid=322, compaction requested=false
2024-11-23T22:56:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46461 {}] regionserver.HRegion(8855): Flush requested on c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:06,473 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-23T22:56:06,477 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into 2026127519424fcbb17442f6c87c3730(size=183.0 K), total size for store is 202.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:06,477 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:06,478 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402566444; duration=0sec
2024-11-23T22:56:06,478 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:06,478 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:06,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/e964f2e809f1438eae15727aa79e2b57 is 1080, key is row0244/info:/1732402566447/Put/seqid=0
2024-11-23T22:56:06,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741877_1053 (size=17918)
2024-11-23T22:56:06,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741877_1053 (size=17918)
2024-11-23T22:56:06,483 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/e964f2e809f1438eae15727aa79e2b57
2024-11-23T22:56:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/e964f2e809f1438eae15727aa79e2b57 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57
2024-11-23T22:56:06,492 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57, entries=12, sequenceid=337, filesize=17.5 K
2024-11-23T22:56:06,493 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=1.05 KB/1076 for c26e66673b075581870eddb8a2594cc9 in 19ms, sequenceid=337, compaction requested=true
2024-11-23T22:56:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c26e66673b075581870eddb8a2594cc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-23T22:56:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:06,493 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-23T22:56:06,494 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 225383 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-23T22:56:06,494 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1541): c26e66673b075581870eddb8a2594cc9/info is initiating minor compaction (all files)
2024-11-23T22:56:06,494 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c26e66673b075581870eddb8a2594cc9/info in TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:06,494 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/2026127519424fcbb17442f6c87c3730, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57] into tmpdir=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp, totalSize=220.1 K
2024-11-23T22:56:06,494 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2026127519424fcbb17442f6c87c3730, keycount=168, bloomtype=ROW, size=183.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732402543986
2024-11-23T22:56:06,495 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73cec92a57354bc281200bd0a8a20540, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732402566418
2024-11-23T22:56:06,495 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e964f2e809f1438eae15727aa79e2b57, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732402566447
2024-11-23T22:56:06,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:06,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:06,507 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c26e66673b075581870eddb8a2594cc9#info#compaction#94 average throughput is 49.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-23T22:56:06,508 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/642151436e8640f68ea639b41b9fba12 is 1080, key is row0062/info:/1732402543986/Put/seqid=0
2024-11-23T22:56:06,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741878_1054 (size=215602)
2024-11-23T22:56:06,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741878_1054 (size=215602)
2024-11-23T22:56:06,516 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/642151436e8640f68ea639b41b9fba12 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/642151436e8640f68ea639b41b9fba12
2024-11-23T22:56:06,521 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c26e66673b075581870eddb8a2594cc9/info of c26e66673b075581870eddb8a2594cc9 into 642151436e8640f68ea639b41b9fba12(size=210.5 K), total size for store is 210.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-23T22:56:06,521 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:06,521 INFO  [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., storeName=c26e66673b075581870eddb8a2594cc9/info, priority=13, startTime=1732402566493; duration=0sec
2024-11-23T22:56:06,521 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-23T22:56:06,521 DEBUG [RS:0;dd96e8dc8369:46461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c26e66673b075581870eddb8a2594cc9:info
2024-11-23T22:56:07,234 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:07,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:07,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:08,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:08,476 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-23T22:56:08,476 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46461%2C1732402530608.1732402568476
2024-11-23T22:56:08,481 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,481 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,481 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,482 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,482 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,482 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402531247 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402568476
2024-11-23T22:56:08,483 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38537:38537),(127.0.0.1/127.0.0.1:37645:37645)]
2024-11-23T22:56:08,483 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402531247 is not closed yet, will try archiving it next time
2024-11-23T22:56:08,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741833_1009 (size=317837)
2024-11-23T22:56:08,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741833_1009 (size=317837)
2024-11-23T22:56:08,487 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing c26e66673b075581870eddb8a2594cc9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-23T22:56:08,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4bae822bd4cb41e581032616ab8f52a7 is 1080, key is row0256/info:/1732402566474/Put/seqid=0
2024-11-23T22:56:08,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:08,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:08,509 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741880_1056 (size=6035)
2024-11-23T22:56:08,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741880_1056 (size=6035)
2024-11-23T22:56:08,510 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4bae822bd4cb41e581032616ab8f52a7
2024-11-23T22:56:08,516 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/.tmp/info/4bae822bd4cb41e581032616ab8f52a7 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4bae822bd4cb41e581032616ab8f52a7
2024-11-23T22:56:08,522 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4bae822bd4cb41e581032616ab8f52a7, entries=1, sequenceid=343, filesize=5.9 K
2024-11-23T22:56:08,523 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c26e66673b075581870eddb8a2594cc9 in 36ms, sequenceid=343, compaction requested=false
2024-11-23T22:56:08,523 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c26e66673b075581870eddb8a2594cc9:

2024-11-23T22:56:08,523 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-23T22:56:08,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/0698e5207bb3442299ed6fc1bc481dd4 is 186, key is TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3./info:regioninfo/1732402547017/Put/seqid=0
2024-11-23T22:56:08,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741881_1057 (size=6153)
2024-11-23T22:56:08,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741881_1057 (size=6153)
2024-11-23T22:56:08,532 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/0698e5207bb3442299ed6fc1bc481dd4
2024-11-23T22:56:08,538 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/.tmp/info/0698e5207bb3442299ed6fc1bc481dd4 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/info/0698e5207bb3442299ed6fc1bc481dd4
2024-11-23T22:56:08,543 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/info/0698e5207bb3442299ed6fc1bc481dd4, entries=5, sequenceid=21, filesize=6.0 K
2024-11-23T22:56:08,544 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false
2024-11-23T22:56:08,544 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-23T22:56:08,545 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fbbb15916746a6267d266e9a976a42a3:

2024-11-23T22:56:08,545 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C46461%2C1732402530608.1732402568545
2024-11-23T22:56:08,549 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,549 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,549 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,549 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,549 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,550 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402568476 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402568545
2024-11-23T22:56:08,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741879_1055 (size=731)
2024-11-23T22:56:08,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741879_1055 (size=731)
2024-11-23T22:56:08,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38537:38537),(127.0.0.1/127.0.0.1:37645:37645)]
2024-11-23T22:56:08,553 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402531247 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs/dd96e8dc8369%2C46461%2C1732402530608.1732402531247
2024-11-23T22:56:08,553 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-23T22:56:08,553 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:56:08,554 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:56:08,554 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:08,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:08,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:08,554 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:56:08,554 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/WALs/dd96e8dc8369,46461,1732402530608/dd96e8dc8369%2C46461%2C1732402530608.1732402568476 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs/dd96e8dc8369%2C46461%2C1732402530608.1732402568476
2024-11-23T22:56:08,554 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:56:08,554 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1567051484, stopped=false
2024-11-23T22:56:08,554 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,35201,1732402530408
2024-11-23T22:56:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:08,647 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:56:08,647 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:56:08,647 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:08,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:08,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:08,648 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,46461,1732402530608' *****
2024-11-23T22:56:08,648 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:56:08,648 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:08,648 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:56:08,649 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(3091): Received CLOSE for c26e66673b075581870eddb8a2594cc9
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(3091): Received CLOSE for fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,46461,1732402530608
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:46461.
2024-11-23T22:56:08,649 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c26e66673b075581870eddb8a2594cc9, disabling compactions & flushes
2024-11-23T22:56:08,649 DEBUG [RS:0;dd96e8dc8369:46461 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:08,649 DEBUG [RS:0;dd96e8dc8369:46461 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:08,649 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:08,649 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:56:08,649 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9. after waiting 0 ms
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:56:08,649 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:56:08,649 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-23T22:56:08,649 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1325): Online Regions={c26e66673b075581870eddb8a2594cc9=TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9., 1588230740=hbase:meta,,1.1588230740, fbbb15916746a6267d266e9a976a42a3=TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.}
2024-11-23T22:56:08,650 DEBUG [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c26e66673b075581870eddb8a2594cc9, fbbb15916746a6267d266e9a976a42a3
2024-11-23T22:56:08,650 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:56:08,650 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:56:08,650 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:56:08,650 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:56:08,650 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:56:08,650 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-top, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/20811710f8154cb4ac426abcc3473227, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7d85aa83298345f68391896d5f5d015e, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e8450c0323f5426fb68028e516bd5743, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/9681708186dc492097da13251c36c0fc, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/dfdf85a03899402dbc6933a3dae4d647, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/d9a3e31391414e83831dde882004906f, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/2026127519424fcbb17442f6c87c3730, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57] to archive
2024-11-23T22:56:08,651 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:56:08,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:56:08,655 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-23T22:56:08,655 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-7cda7c9671dc421db7c541de75cc893e
2024-11-23T22:56:08,655 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:56:08,655 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:56:08,656 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402568650Running coprocessor pre-close hooks at 1732402568650Disabling compacts and flushes for region at 1732402568650Disabling writes for close at 1732402568650Writing region close event to WAL at 1732402568651 (+1 ms)Running coprocessor post-close hooks at 1732402568655 (+4 ms)Closed at 1732402568655
2024-11-23T22:56:08,656 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:56:08,657 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/20811710f8154cb4ac426abcc3473227 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/20811710f8154cb4ac426abcc3473227
2024-11-23T22:56:08,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/TestLogRolling-testLogRolling=0a51d8942b5b3c2e061b44a1459cc889-edb018425fdf4abd97aa4366c9be5854
2024-11-23T22:56:08,659 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/27f92ca0aa1142a7a7bd3ce1366aec8b
2024-11-23T22:56:08,660 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7d85aa83298345f68391896d5f5d015e to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7d85aa83298345f68391896d5f5d015e
2024-11-23T22:56:08,661 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7a6e165240074fc2bd8a28b01edb0e16
2024-11-23T22:56:08,662 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/4cf69d5cd0c548609da623535c4d648b
2024-11-23T22:56:08,663 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e8450c0323f5426fb68028e516bd5743 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e8450c0323f5426fb68028e516bd5743
2024-11-23T22:56:08,664 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/5c9f1e06d6e14998a44837f8838a7017
2024-11-23T22:56:08,665 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/83bbf8384d81451b8b2e93088e334c0d
2024-11-23T22:56:08,667 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/9681708186dc492097da13251c36c0fc to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/9681708186dc492097da13251c36c0fc
2024-11-23T22:56:08,668 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/78eaf389fffa407ba5f73a98888da8e2
2024-11-23T22:56:08,669 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/81fd70822fe14b40be0f1241d30fdb70
2024-11-23T22:56:08,670 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/dfdf85a03899402dbc6933a3dae4d647 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/dfdf85a03899402dbc6933a3dae4d647
2024-11-23T22:56:08,671 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/7c1c332e14bd4cd8ace639b243f8c456
2024-11-23T22:56:08,672 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/09cc36aa228043568c6e55f8b12ebabd
2024-11-23T22:56:08,673 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/d9a3e31391414e83831dde882004906f to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/d9a3e31391414e83831dde882004906f
2024-11-23T22:56:08,675 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/bd6c85d2de674b6088f01b2e6c0bf783
2024-11-23T22:56:08,676 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/08015e54b7804fe68d6e5ca5d9305dbc
2024-11-23T22:56:08,677 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/2026127519424fcbb17442f6c87c3730 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/2026127519424fcbb17442f6c87c3730
2024-11-23T22:56:08,678 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/b418cd67fe604c08a8eb522e2b86c05d
2024-11-23T22:56:08,679 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/73cec92a57354bc281200bd0a8a20540
2024-11-23T22:56:08,680 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/info/e964f2e809f1438eae15727aa79e2b57
2024-11-23T22:56:08,681 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=dd96e8dc8369:35201 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-23T22:56:08,681 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [20811710f8154cb4ac426abcc3473227=42984, 27f92ca0aa1142a7a7bd3ce1366aec8b=12516, 7d85aa83298345f68391896d5f5d015e=64714, 7a6e165240074fc2bd8a28b01edb0e16=19000, 4cf69d5cd0c548609da623535c4d648b=16828, e8450c0323f5426fb68028e516bd5743=84293, 5c9f1e06d6e14998a44837f8838a7017=12516, 83bbf8384d81451b8b2e93088e334c0d=17906, 9681708186dc492097da13251c36c0fc=113509, 78eaf389fffa407ba5f73a98888da8e2=21156, 81fd70822fe14b40be0f1241d30fdb70=16828, dfdf85a03899402dbc6933a3dae4d647=139688, 7c1c332e14bd4cd8ace639b243f8c456=19000, 09cc36aa228043568c6e55f8b12ebabd=13595, d9a3e31391414e83831dde882004906f=165683, bd6c85d2de674b6088f01b2e6c0bf783=22254, 08015e54b7804fe68d6e5ca5d9305dbc=19013, 2026127519424fcbb17442f6c87c3730=187373, b418cd67fe604c08a8eb522e2b86c05d=12523, 73cec92a57354bc281200bd0a8a20540=20092, e964f2e809f1438eae15727aa79e2b57=17918]
2024-11-23T22:56:08,685 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/c26e66673b075581870eddb8a2594cc9/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130
2024-11-23T22:56:08,685 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c26e66673b075581870eddb8a2594cc9:
Waiting for close lock at 1732402568649Running coprocessor pre-close hooks at 1732402568649Disabling compacts and flushes for region at 1732402568649Disabling writes for close at 1732402568649Writing region close event to WAL at 1732402568682 (+33 ms)Running coprocessor post-close hooks at 1732402568685 (+3 ms)Closed at 1732402568685
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732402546096.c26e66673b075581870eddb8a2594cc9.
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbbb15916746a6267d266e9a976a42a3, disabling compactions & flushes
2024-11-23T22:56:08,686 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3. after waiting 0 ms
2024-11-23T22:56:08,686 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:56:08,686 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889->hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/0a51d8942b5b3c2e061b44a1459cc889/info/4c1dd610c4c54bb4bf368c95871becf4-bottom] to archive
2024-11-23T22:56:08,687 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-23T22:56:08,689 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889 to hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/archive/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/info/4c1dd610c4c54bb4bf368c95871becf4.0a51d8942b5b3c2e061b44a1459cc889
2024-11-23T22:56:08,689 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-23T22:56:08,692 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/data/default/TestLogRolling-testLogRolling/fbbb15916746a6267d266e9a976a42a3/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-23T22:56:08,693 INFO  [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:56:08,693 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbbb15916746a6267d266e9a976a42a3:
Waiting for close lock at 1732402568686Running coprocessor pre-close hooks at 1732402568686Disabling compacts and flushes for region at 1732402568686Disabling writes for close at 1732402568686Writing region close event to WAL at 1732402568689 (+3 ms)Running coprocessor post-close hooks at 1732402568693 (+4 ms)Closed at 1732402568693
2024-11-23T22:56:08,693 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732402546096.fbbb15916746a6267d266e9a976a42a3.
2024-11-23T22:56:08,850 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,46461,1732402530608; all regions closed.
2024-11-23T22:56:08,850 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,850 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,850 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,850 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,851 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741834_1010 (size=8107)
2024-11-23T22:56:08,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741834_1010 (size=8107)
2024-11-23T22:56:08,855 DEBUG [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs
2024-11-23T22:56:08,855 INFO  [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C46461%2C1732402530608.meta:.meta(num 1732402531659)
2024-11-23T22:56:08,856 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,856 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,856 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,856 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,856 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:08,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741882_1058 (size=780)
2024-11-23T22:56:08,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741882_1058 (size=780)
2024-11-23T22:56:08,861 DEBUG [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/oldWALs
2024-11-23T22:56:08,861 INFO  [RS:0;dd96e8dc8369:46461 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C46461%2C1732402530608:(num 1732402568545)
2024-11-23T22:56:08,861 DEBUG [RS:0;dd96e8dc8369:46461 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:08,861 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:56:08,861 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:56:08,861 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:56:08,861 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:56:08,862 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:56:08,862 INFO  [RS:0;dd96e8dc8369:46461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46461
2024-11-23T22:56:08,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,46461,1732402530608
2024-11-23T22:56:08,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:56:08,871 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:56:08,882 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,46461,1732402530608]
2024-11-23T22:56:08,892 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,46461,1732402530608 already deleted, retry=false
2024-11-23T22:56:08,892 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,46461,1732402530608 expired; onlineServers=0
2024-11-23T22:56:08,892 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,35201,1732402530408' *****
2024-11-23T22:56:08,892 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:56:08,892 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:56:08,892 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:56:08,893 DEBUG [M:0;dd96e8dc8369:35201 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:56:08,893 DEBUG [M:0;dd96e8dc8369:35201 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:56:08,893 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:56:08,893 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402531020 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402531020,5,FailOnTimeoutGroup]
2024-11-23T22:56:08,893 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402531024 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402531024,5,FailOnTimeoutGroup]
2024-11-23T22:56:08,893 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:56:08,893 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:56:08,893 DEBUG [M:0;dd96e8dc8369:35201 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:56:08,893 INFO  [M:0;dd96e8dc8369:35201 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:56:08,893 INFO  [M:0;dd96e8dc8369:35201 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:56:08,894 INFO  [M:0;dd96e8dc8369:35201 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:56:08,894 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:56:08,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:56:08,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:08,903 DEBUG [M:0;dd96e8dc8369:35201 {}] zookeeper.ZKUtil(347): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:56:08,903 WARN  [M:0;dd96e8dc8369:35201 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:56:08,904 INFO  [M:0;dd96e8dc8369:35201 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/.lastflushedseqids
2024-11-23T22:56:08,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741883_1059 (size=228)
2024-11-23T22:56:08,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741883_1059 (size=228)
2024-11-23T22:56:08,911 INFO  [M:0;dd96e8dc8369:35201 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:56:08,911 INFO  [M:0;dd96e8dc8369:35201 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:56:08,911 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:56:08,911 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:08,911 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:08,911 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:56:08,911 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:08,911 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.41 KB heapSize=63.35 KB
2024-11-23T22:56:08,928 DEBUG [M:0;dd96e8dc8369:35201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20ca9232b99b420ab01ff31baf22d9d4 is 82, key is hbase:meta,,1/info:regioninfo/1732402531685/Put/seqid=0
2024-11-23T22:56:08,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741884_1060 (size=5672)
2024-11-23T22:56:08,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741884_1060 (size=5672)
2024-11-23T22:56:08,933 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20ca9232b99b420ab01ff31baf22d9d4
2024-11-23T22:56:08,951 DEBUG [M:0;dd96e8dc8369:35201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb66a9b888894cdab1abf2310707fe57 is 748, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402532132/Put/seqid=0
2024-11-23T22:56:08,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741885_1061 (size=7088)
2024-11-23T22:56:08,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741885_1061 (size=7088)
2024-11-23T22:56:08,956 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.80 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb66a9b888894cdab1abf2310707fe57
2024-11-23T22:56:08,960 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb66a9b888894cdab1abf2310707fe57
2024-11-23T22:56:08,975 DEBUG [M:0;dd96e8dc8369:35201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a9fad58d8ff4df98e71dbda3f98df7c is 69, key is dd96e8dc8369,46461,1732402530608/rs:state/1732402531078/Put/seqid=0
2024-11-23T22:56:08,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741886_1062 (size=5156)
2024-11-23T22:56:08,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741886_1062 (size=5156)
2024-11-23T22:56:08,979 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a9fad58d8ff4df98e71dbda3f98df7c
2024-11-23T22:56:08,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:08,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46461-0x10169e8c4010001, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:08,988 INFO  [RS:0;dd96e8dc8369:46461 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:56:08,988 INFO  [RS:0;dd96e8dc8369:46461 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,46461,1732402530608; zookeeper connection closed.
2024-11-23T22:56:08,988 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@598d617 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@598d617
2024-11-23T22:56:08,988 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:56:08,998 DEBUG [M:0;dd96e8dc8369:35201 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/325e0ba3382a4b988c9ce957a242dba6 is 52, key is load_balancer_on/state:d/1732402531759/Put/seqid=0
2024-11-23T22:56:09,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741887_1063 (size=5056)
2024-11-23T22:56:09,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741887_1063 (size=5056)
2024-11-23T22:56:09,003 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/325e0ba3382a4b988c9ce957a242dba6
2024-11-23T22:56:09,008 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20ca9232b99b420ab01ff31baf22d9d4 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20ca9232b99b420ab01ff31baf22d9d4
2024-11-23T22:56:09,013 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20ca9232b99b420ab01ff31baf22d9d4, entries=8, sequenceid=125, filesize=5.5 K
2024-11-23T22:56:09,013 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb66a9b888894cdab1abf2310707fe57 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb66a9b888894cdab1abf2310707fe57
2024-11-23T22:56:09,019 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb66a9b888894cdab1abf2310707fe57
2024-11-23T22:56:09,019 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb66a9b888894cdab1abf2310707fe57, entries=13, sequenceid=125, filesize=6.9 K
2024-11-23T22:56:09,020 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a9fad58d8ff4df98e71dbda3f98df7c as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a9fad58d8ff4df98e71dbda3f98df7c
2024-11-23T22:56:09,025 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a9fad58d8ff4df98e71dbda3f98df7c, entries=1, sequenceid=125, filesize=5.0 K
2024-11-23T22:56:09,026 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/325e0ba3382a4b988c9ce957a242dba6 as hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/325e0ba3382a4b988c9ce957a242dba6
2024-11-23T22:56:09,030 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39749/user/jenkins/test-data/b4800294-eeeb-0996-c67b-a7bff1b0b392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/325e0ba3382a4b988c9ce957a242dba6, entries=1, sequenceid=125, filesize=4.9 K
2024-11-23T22:56:09,035 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=125, compaction requested=false
2024-11-23T22:56:09,036 INFO  [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:09,036 DEBUG [M:0;dd96e8dc8369:35201 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402568911Disabling compacts and flushes for region at 1732402568911Disabling writes for close at 1732402568911Obtaining lock to block concurrent updates at 1732402568911Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402568911Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52639, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1732402568912 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402568912Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402568913 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402568928 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402568928Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402568937 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402568951 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402568951Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402568960 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402568974 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402568974Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402568983 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402568998 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402568998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15e14e35: reopening flushed file at 1732402569007 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@366b4122: reopening flushed file at 1732402569013 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a46a8b: reopening flushed file at 1732402569019 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@487712e: reopening flushed file at 1732402569025 (+6 ms)Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=125, compaction requested=false at 1732402569035 (+10 ms)Writing region close event to WAL at 1732402569036 (+1 ms)Closed at 1732402569036
2024-11-23T22:56:09,037 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:09,037 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:09,037 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:09,037 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:09,037 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:09,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34353 is added to blk_1073741830_1006 (size=61308)
2024-11-23T22:56:09,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34181 is added to blk_1073741830_1006 (size=61308)
2024-11-23T22:56:09,040 INFO  [M:0;dd96e8dc8369:35201 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:56:09,040 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:56:09,040 INFO  [M:0;dd96e8dc8369:35201 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35201
2024-11-23T22:56:09,040 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:56:09,108 INFO  [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:56:09,150 INFO  [M:0;dd96e8dc8369:35201 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:56:09,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:09,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35201-0x10169e8c4010000, quorum=127.0.0.1:56865, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:09,153 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ec1477d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:09,153 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5169cded{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:09,153 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:09,153 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68b06d6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:09,153 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b9fec83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:09,155 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:56:09,155 WARN  [BP-2077918267-172.17.0.2-1732402527974 heartbeating to localhost/127.0.0.1:39749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:56:09,155 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:56:09,155 WARN  [BP-2077918267-172.17.0.2-1732402527974 heartbeating to localhost/127.0.0.1:39749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2077918267-172.17.0.2-1732402527974 (Datanode Uuid e39ac941-01e7-46a1-b8f1-10c1ab73d3ba) service to localhost/127.0.0.1:39749
2024-11-23T22:56:09,156 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data3/current/BP-2077918267-172.17.0.2-1732402527974 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:09,156 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data4/current/BP-2077918267-172.17.0.2-1732402527974 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:09,156 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:56:09,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c8f3fae{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:09,158 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43cde2d6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:09,158 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:09,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@98b6596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:09,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7edf9828{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:09,160 WARN  [BP-2077918267-172.17.0.2-1732402527974 heartbeating to localhost/127.0.0.1:39749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:56:09,160 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:56:09,160 WARN  [BP-2077918267-172.17.0.2-1732402527974 heartbeating to localhost/127.0.0.1:39749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2077918267-172.17.0.2-1732402527974 (Datanode Uuid b3db603e-c19e-4ee9-87b8-748baac1eec7) service to localhost/127.0.0.1:39749
2024-11-23T22:56:09,160 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:56:09,160 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data1/current/BP-2077918267-172.17.0.2-1732402527974 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:09,160 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/cluster_dcc47dde-64cd-759c-6056-0e6f9ae53370/data/data2/current/BP-2077918267-172.17.0.2-1732402527974 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:09,160 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:56:09,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bb84093{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:56:09,166 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49a552fc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:09,166 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:09,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e27d2b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:09,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32fa876c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:09,173 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:56:09,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:56:09,223 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=233 (was 209)
Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39749
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39749
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39749 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39749
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:39749
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39749 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39749
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39749 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=512 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6287 (was 6471)
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=233, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=6288
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.log.dir so I do NOT create it in target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/343f8187-b886-be49-2d81-2468d21421b0/hadoop.tmp.dir so I do NOT create it in target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d, deleteOnExit=true
2024-11-23T22:56:09,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/test.cache.data in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.tmp.dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-23T22:56:09,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:56:09,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/nfs.dump.dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/java.io.tmpdir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-23T22:56:09,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-23T22:56:09,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:09,250 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:56:09,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:09,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:09,620 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:56:09,624 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:56:09,625 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:56:09,625 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:56:09,625 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:56:09,626 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:56:09,626 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ee84893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:56:09,627 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6833f9d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:56:09,728 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39cd0eab{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/java.io.tmpdir/jetty-localhost-36273-hadoop-hdfs-3_4_1-tests_jar-_-any-12669433211482381664/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:56:09,729 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4833305a{HTTP/1.1, (http/1.1)}{localhost:36273}
2024-11-23T22:56:09,729 INFO  [Time-limited test {}] server.Server(415): Started @295054ms
2024-11-23T22:56:09,741 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-23T22:56:10,004 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:56:10,008 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:56:10,009 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:56:10,009 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:56:10,009 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-23T22:56:10,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29692e24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:56:10,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5139465a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:56:10,149 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c04d24a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/java.io.tmpdir/jetty-localhost-36443-hadoop-hdfs-3_4_1-tests_jar-_-any-17542723804665967396/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:10,149 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@550c39ff{HTTP/1.1, (http/1.1)}{localhost:36443}
2024-11-23T22:56:10,149 INFO  [Time-limited test {}] server.Server(415): Started @295474ms
2024-11-23T22:56:10,151 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:56:10,205 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-23T22:56:10,208 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-23T22:56:10,216 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-23T22:56:10,216 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-23T22:56:10,216 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-23T22:56:10,217 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2074d4e0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,AVAILABLE}
2024-11-23T22:56:10,217 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d140981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-23T22:56:10,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:10,355 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11e68b55{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/java.io.tmpdir/jetty-localhost-34421-hadoop-hdfs-3_4_1-tests_jar-_-any-16963256114639253009/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:10,355 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@740bac32{HTTP/1.1, (http/1.1)}{localhost:34421}
2024-11-23T22:56:10,355 INFO  [Time-limited test {}] server.Server(415): Started @295680ms
2024-11-23T22:56:10,357 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-23T22:56:10,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:10,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:11,188 WARN  [Thread-2525 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data2/current/BP-2361242-172.17.0.2-1732402569254/current, will proceed with Du for space computation calculation, 
2024-11-23T22:56:11,188 WARN  [Thread-2524 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data1/current/BP-2361242-172.17.0.2-1732402569254/current, will proceed with Du for space computation calculation, 
2024-11-23T22:56:11,229 WARN  [Thread-2489 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:56:11,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:11,238 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7e831bfa2dfaeda with lease ID 0x1fb63680a777f5cd: Processing first storage report for DS-5614423f-4733-468b-b7e1-6b86086aa05f from datanode DatanodeRegistration(127.0.0.1:40607, datanodeUuid=b876a378-4509-46aa-b0cb-73a66058756b, infoPort=36991, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254)
2024-11-23T22:56:11,238 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7e831bfa2dfaeda with lease ID 0x1fb63680a777f5cd: from storage DS-5614423f-4733-468b-b7e1-6b86086aa05f node DatanodeRegistration(127.0.0.1:40607, datanodeUuid=b876a378-4509-46aa-b0cb-73a66058756b, infoPort=36991, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:56:11,238 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7e831bfa2dfaeda with lease ID 0x1fb63680a777f5cd: Processing first storage report for DS-52ed831c-1524-4ef8-8df3-adde983ab2fe from datanode DatanodeRegistration(127.0.0.1:40607, datanodeUuid=b876a378-4509-46aa-b0cb-73a66058756b, infoPort=36991, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254)
2024-11-23T22:56:11,238 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7e831bfa2dfaeda with lease ID 0x1fb63680a777f5cd: from storage DS-52ed831c-1524-4ef8-8df3-adde983ab2fe node DatanodeRegistration(127.0.0.1:40607, datanodeUuid=b876a378-4509-46aa-b0cb-73a66058756b, infoPort=36991, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:56:11,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:11,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:11,646 WARN  [Thread-2536 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data3/current/BP-2361242-172.17.0.2-1732402569254/current, will proceed with Du for space computation calculation, 
2024-11-23T22:56:11,646 WARN  [Thread-2537 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data4/current/BP-2361242-172.17.0.2-1732402569254/current, will proceed with Du for space computation calculation, 
2024-11-23T22:56:11,665 WARN  [Thread-2512 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-23T22:56:11,667 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab0790cb51b2f8ea with lease ID 0x1fb63680a777f5ce: Processing first storage report for DS-cb926c9e-d2f2-477d-b6d7-ee972a44aed8 from datanode DatanodeRegistration(127.0.0.1:36735, datanodeUuid=04ea62da-9eed-41b4-9a91-441b2c672e61, infoPort=46709, infoSecurePort=0, ipcPort=41155, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254)
2024-11-23T22:56:11,667 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab0790cb51b2f8ea with lease ID 0x1fb63680a777f5ce: from storage DS-cb926c9e-d2f2-477d-b6d7-ee972a44aed8 node DatanodeRegistration(127.0.0.1:36735, datanodeUuid=04ea62da-9eed-41b4-9a91-441b2c672e61, infoPort=46709, infoSecurePort=0, ipcPort=41155, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:56:11,667 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab0790cb51b2f8ea with lease ID 0x1fb63680a777f5ce: Processing first storage report for DS-55a58c14-e10d-42f9-a960-b88a64a75344 from datanode DatanodeRegistration(127.0.0.1:36735, datanodeUuid=04ea62da-9eed-41b4-9a91-441b2c672e61, infoPort=46709, infoSecurePort=0, ipcPort=41155, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254)
2024-11-23T22:56:11,667 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab0790cb51b2f8ea with lease ID 0x1fb63680a777f5ce: from storage DS-55a58c14-e10d-42f9-a960-b88a64a75344 node DatanodeRegistration(127.0.0.1:36735, datanodeUuid=04ea62da-9eed-41b4-9a91-441b2c672e61, infoPort=46709, infoSecurePort=0, ipcPort=41155, storageInfo=lv=-57;cid=testClusterID;nsid=1604377714;c=1732402569254), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-23T22:56:11,695 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2
2024-11-23T22:56:11,702 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/zookeeper_0, clientPort=59848, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-23T22:56:11,703 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59848
2024-11-23T22:56:11,703 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:11,705 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:11,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:56:11,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741825_1001 (size=7)
2024-11-23T22:56:11,723 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb with version=8
2024-11-23T22:56:11,723 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35437/user/jenkins/test-data/459cec4b-1519-ab60-a45d-b7cf5ec023b8/hbase-staging
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:56:11,725 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-23T22:56:11,726 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:56:11,726 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35761
2024-11-23T22:56:11,727 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35761 connecting to ZooKeeper ensemble=127.0.0.1:59848
2024-11-23T22:56:11,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357610x0, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:56:11,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35761-0x10169e965600000 connected
2024-11-23T22:56:11,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:56:11,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-23T22:56:11,913 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-23T22:56:11,914 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-23T22:56:11,966 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:11,968 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:11,970 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:11,971 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb, hbase.cluster.distributed=false
2024-11-23T22:56:11,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:56:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35761
2024-11-23T22:56:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35761
2024-11-23T22:56:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35761
2024-11-23T22:56:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35761
2024-11-23T22:56:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35761
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-23T22:56:11,990 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-23T22:56:11,991 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-23T22:56:11,991 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-23T22:56:11,992 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34101
2024-11-23T22:56:11,993 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34101 connecting to ZooKeeper ensemble=127.0.0.1:59848
2024-11-23T22:56:11,994 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:11,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:12,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341010x0, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-23T22:56:12,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341010x0, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:12,008 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34101-0x10169e965600001 connected
2024-11-23T22:56:12,009 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-23T22:56:12,009 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-23T22:56:12,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-23T22:56:12,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-23T22:56:12,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34101
2024-11-23T22:56:12,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34101
2024-11-23T22:56:12,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34101
2024-11-23T22:56:12,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34101
2024-11-23T22:56:12,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34101
2024-11-23T22:56:12,027 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:35761
2024-11-23T22:56:12,028 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:56:12,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:56:12,097 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-23T22:56:12,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,162 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-23T22:56:12,163 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,35761,1732402571725 from backup master directory
2024-11-23T22:56:12,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:56:12,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-23T22:56:12,187 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:56:12,187 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,190 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/hbase.id] with ID: e7276fe1-58e1-4471-9891-b975e6403888
2024-11-23T22:56:12,190 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/.tmp/hbase.id
2024-11-23T22:56:12,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:56:12,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741826_1002 (size=42)
2024-11-23T22:56:12,196 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/.tmp/hbase.id]:[hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/hbase.id]
2024-11-23T22:56:12,208 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:12,208 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-23T22:56:12,209 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-23T22:56:12,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:56:12,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741827_1003 (size=196)
2024-11-23T22:56:12,237 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:12,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:12,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:12,627 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-23T22:56:12,628 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-23T22:56:12,628 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:56:12,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:56:12,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741828_1004 (size=1189)
2024-11-23T22:56:12,637 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store
2024-11-23T22:56:12,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:56:12,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741829_1005 (size=34)
2024-11-23T22:56:12,644 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:56:12,645 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:56:12,645 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:12,645 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:12,645 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:56:12,645 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:12,645 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:12,645 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402572645Disabling compacts and flushes for region at 1732402572645Disabling writes for close at 1732402572645Writing region close event to WAL at 1732402572645Closed at 1732402572645
2024-11-23T22:56:12,646 WARN  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/.initializing
2024-11-23T22:56:12,646 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/WALs/dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,648 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C35761%2C1732402571725, suffix=, logDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/WALs/dd96e8dc8369,35761,1732402571725, archiveDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/oldWALs, maxLogs=10
2024-11-23T22:56:12,648 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C35761%2C1732402571725.1732402572648
2024-11-23T22:56:12,667 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/WALs/dd96e8dc8369,35761,1732402571725/dd96e8dc8369%2C35761%2C1732402571725.1732402572648
2024-11-23T22:56:12,668 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36991:36991),(127.0.0.1/127.0.0.1:46709:46709)]
2024-11-23T22:56:12,672 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:56:12,673 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:56:12,673 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,673 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,674 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:56:12,675 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-23T22:56:12,675 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,676 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:12,676 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:56:12,677 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-23T22:56:12,677 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,677 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:56:12,677 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:56:12,679 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-23T22:56:12,679 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,679 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:56:12,679 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-23T22:56:12,680 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-23T22:56:12,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,681 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-23T22:56:12,681 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,681 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,682 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,683 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,683 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,683 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-23T22:56:12,684 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-23T22:56:12,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:56:12,686 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814892, jitterRate=0.036188989877700806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-23T22:56:12,687 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732402572673Initializing all the Stores at 1732402572674 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402572674Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402572674Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402572674Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402572674Cleaning up temporary data from old regions at 1732402572683 (+9 ms)Region opened successfully at 1732402572687 (+4 ms)
2024-11-23T22:56:12,687 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-23T22:56:12,690 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13271130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:56:12,691 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-23T22:56:12,691 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-23T22:56:12,691 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-23T22:56:12,691 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-23T22:56:12,692 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-23T22:56:12,692 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-23T22:56:12,692 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-23T22:56:12,694 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-23T22:56:12,695 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-23T22:56:12,745 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-23T22:56:12,745 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-23T22:56:12,746 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-23T22:56:12,755 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-23T22:56:12,756 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-23T22:56:12,758 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-23T22:56:12,771 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-23T22:56:12,771 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-23T22:56:12,783 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-23T22:56:12,786 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-23T22:56:12,797 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-23T22:56:12,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:12,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:12,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,808 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,35761,1732402571725, sessionid=0x10169e965600000, setting cluster-up flag (Was=false)
2024-11-23T22:56:12,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,861 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-23T22:56:12,863 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:12,913 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-23T22:56:12,916 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:12,918 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-23T22:56:12,920 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-23T22:56:12,921 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-23T22:56:12,921 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-23T22:56:12,921 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,35761,1732402571725
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:56:12,924 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402602925
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-23T22:56:12,926 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:12,927 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:56:12,927 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-23T22:56:12,927 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-23T22:56:12,927 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-23T22:56:12,927 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-23T22:56:12,928 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402572928,5,FailOnTimeoutGroup]
2024-11-23T22:56:12,928 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,928 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402572928,5,FailOnTimeoutGroup]
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:12,928 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:12,928 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-23T22:56:12,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:56:12,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741831_1007 (size=1321)
2024-11-23T22:56:12,935 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-23T22:56:12,936 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb
2024-11-23T22:56:12,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:56:12,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741832_1008 (size=32)
2024-11-23T22:56:12,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:56:12,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:56:12,946 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:56:12,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:12,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:56:12,948 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:56:12,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:12,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:56:12,951 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:56:12,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:12,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:56:12,953 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:56:12,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:12,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:12,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:56:12,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740
2024-11-23T22:56:12,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740
2024-11-23T22:56:12,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:56:12,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:56:12,956 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:56:12,957 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:56:12,959 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-23T22:56:12,959 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692869, jitterRate=-0.11897237598896027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732402572943Initializing all the Stores at 1732402572944 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402572944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402572945 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402572945Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402572945Cleaning up temporary data from old regions at 1732402572956 (+11 ms)Region opened successfully at 1732402572960 (+4 ms)
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:56:12,960 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:56:12,960 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:56:12,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402572960Disabling compacts and flushes for region at 1732402572960Disabling writes for close at 1732402572960Writing region close event to WAL at 1732402572960Closed at 1732402572960
2024-11-23T22:56:12,961 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:56:12,961 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-23T22:56:12,961 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-23T22:56:12,962 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:56:12,963 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-23T22:56:13,016 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(746): ClusterId : e7276fe1-58e1-4471-9891-b975e6403888
2024-11-23T22:56:13,016 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-23T22:56:13,032 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-23T22:56:13,032 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-23T22:56:13,040 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-23T22:56:13,040 DEBUG [RS:0;dd96e8dc8369:34101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8879e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0
2024-11-23T22:56:13,055 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:34101
2024-11-23T22:56:13,056 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-23T22:56:13,056 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-23T22:56:13,056 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-23T22:56:13,056 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,35761,1732402571725 with port=34101, startcode=1732402571990
2024-11-23T22:56:13,057 DEBUG [RS:0;dd96e8dc8369:34101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-23T22:56:13,059 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33797, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-23T22:56:13,059 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35761 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,059 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35761 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,060 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb
2024-11-23T22:56:13,060 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46691
2024-11-23T22:56:13,060 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-23T22:56:13,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:56:13,071 DEBUG [RS:0;dd96e8dc8369:34101 {}] zookeeper.ZKUtil(111): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,072 WARN  [RS:0;dd96e8dc8369:34101 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-23T22:56:13,072 INFO  [RS:0;dd96e8dc8369:34101 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:56:13,072 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,072 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,34101,1732402571990]
2024-11-23T22:56:13,075 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-23T22:56:13,077 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-23T22:56:13,077 INFO  [RS:0;dd96e8dc8369:34101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-23T22:56:13,077 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,078 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-23T22:56:13,078 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-23T22:56:13,078 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:56:13,079 DEBUG [RS:0;dd96e8dc8369:34101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,081 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,34101,1732402571990-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:56:13,097 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-23T22:56:13,097 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,34101,1732402571990-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,097 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,097 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.Replication(171): dd96e8dc8369,34101,1732402571990 started
2024-11-23T22:56:13,110 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,110 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,34101,1732402571990, RpcServer on dd96e8dc8369/172.17.0.2:34101, sessionid=0x10169e965600001
2024-11-23T22:56:13,110 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-23T22:56:13,110 DEBUG [RS:0;dd96e8dc8369:34101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,110 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,34101,1732402571990'
2024-11-23T22:56:13,110 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-23T22:56:13,110 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,34101,1732402571990'
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-23T22:56:13,111 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-23T22:56:13,112 DEBUG [RS:0;dd96e8dc8369:34101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-23T22:56:13,112 INFO  [RS:0;dd96e8dc8369:34101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-23T22:56:13,112 INFO  [RS:0;dd96e8dc8369:34101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-23T22:56:13,113 WARN  [dd96e8dc8369:35761 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-23T22:56:13,213 INFO  [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C34101%2C1732402571990, suffix=, logDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/dd96e8dc8369,34101,1732402571990, archiveDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs, maxLogs=32
2024-11-23T22:56:13,214 INFO  [RS:0;dd96e8dc8369:34101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C34101%2C1732402571990.1732402573214
2024-11-23T22:56:13,219 INFO  [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/dd96e8dc8369,34101,1732402571990/dd96e8dc8369%2C34101%2C1732402571990.1732402573214
2024-11-23T22:56:13,220 DEBUG [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46709:46709),(127.0.0.1/127.0.0.1:36991:36991)]
2024-11-23T22:56:13,237 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:13,363 DEBUG [dd96e8dc8369:35761 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-23T22:56:13,364 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,365 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,34101,1732402571990, state=OPENING
2024-11-23T22:56:13,425 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-23T22:56:13,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:13,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:13,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:56:13,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:56:13,437 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-23T22:56:13,437 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,34101,1732402571990}]
2024-11-23T22:56:13,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:13,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:13,590 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-23T22:56:13,592 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-23T22:56:13,596 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-23T22:56:13,596 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:56:13,599 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C34101%2C1732402571990.meta, suffix=.meta, logDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/dd96e8dc8369,34101,1732402571990, archiveDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs, maxLogs=32
2024-11-23T22:56:13,599 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C34101%2C1732402571990.meta.1732402573599.meta
2024-11-23T22:56:13,613 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/dd96e8dc8369,34101,1732402571990/dd96e8dc8369%2C34101%2C1732402571990.meta.1732402573599.meta
2024-11-23T22:56:13,619 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46709:46709),(127.0.0.1/127.0.0.1:36991:36991)]
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-23T22:56:13,626 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-23T22:56:13,626 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-23T22:56:13,628 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-23T22:56:13,628 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-23T22:56:13,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:13,629 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:13,629 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-23T22:56:13,630 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-23T22:56:13,630 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:13,630 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:13,630 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-23T22:56:13,631 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-23T22:56:13,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:13,631 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:13,631 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-23T22:56:13,632 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-23T22:56:13,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-23T22:56:13,632 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-23T22:56:13,632 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-23T22:56:13,633 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740
2024-11-23T22:56:13,634 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740
2024-11-23T22:56:13,635 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-23T22:56:13,635 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-23T22:56:13,635 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-23T22:56:13,636 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-23T22:56:13,637 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792833, jitterRate=0.008140206336975098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-23T22:56:13,637 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-23T22:56:13,637 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732402573626Writing region info on filesystem at 1732402573626Initializing all the Stores at 1732402573627 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402573627Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402573627Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402573627Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402573627Cleaning up temporary data from old regions at 1732402573635 (+8 ms)Running coprocessor post-open hooks at 1732402573637 (+2 ms)Region opened successfully at 1732402573637
2024-11-23T22:56:13,638 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402573590
2024-11-23T22:56:13,640 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-23T22:56:13,640 INFO  [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-23T22:56:13,641 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,642 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,34101,1732402571990, state=OPEN
2024-11-23T22:56:13,656 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,656 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,656 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,657 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,657 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,657 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:56:13,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-23T22:56:13,674 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:13,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:56:13,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-23T22:56:13,676 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-23T22:56:13,676 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,34101,1732402571990 in 237 msec
2024-11-23T22:56:13,678 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-23T22:56:13,678 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec
2024-11-23T22:56:13,679 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-23T22:56:13,679 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-23T22:56:13,680 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:56:13,680 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,34101,1732402571990, seqNum=-1]
2024-11-23T22:56:13,680 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:56:13,681 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:56:13,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,686 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 766 msec
2024-11-23T22:56:13,686 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402573686, completionTime=-1
2024-11-23T22:56:13,686 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-23T22:56:13,686 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-23T22:56:13,688 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402633689
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402693689
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:35761, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,689 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,691 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,691 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,691 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.506sec
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-23T22:56:13,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-23T22:56:13,693 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-23T22:56:13,696 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-23T22:56:13,696 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-23T22:56:13,696 INFO  [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35761,1732402571725-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-23T22:56:13,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f27634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:56:13,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,35761,-1 for getting cluster id
2024-11-23T22:56:13,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-23T22:56:13,717 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7276fe1-58e1-4471-9891-b975e6403888'
2024-11-23T22:56:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-23T22:56:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7276fe1-58e1-4471-9891-b975e6403888"

2024-11-23T22:56:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e132b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:56:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,35761,-1]
2024-11-23T22:56:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-23T22:56:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:13,719 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-23T22:56:13,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d77e1a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-23T22:56:13,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-23T22:56:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,34101,1732402571990, seqNum=-1]
2024-11-23T22:56:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-23T22:56:13,722 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-23T22:56:13,724 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:13,724 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-23T22:56:13,726 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-23T22:56:13,727 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-23T22:56:13,729 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1, archiveDir=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs, maxLogs=32
2024-11-23T22:56:13,729 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732402573729
2024-11-23T22:56:13,734 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1/test.com%2C8080%2C1.1732402573729
2024-11-23T22:56:13,735 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46709:46709),(127.0.0.1/127.0.0.1:36991:36991)]
2024-11-23T22:56:13,740 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732402573740
2024-11-23T22:56:13,746 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,746 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,746 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,746 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,746 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,746 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1/test.com%2C8080%2C1.1732402573729 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1/test.com%2C8080%2C1.1732402573740
2024-11-23T22:56:13,749 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36991:36991),(127.0.0.1/127.0.0.1:46709:46709)]
2024-11-23T22:56:13,749 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1/test.com%2C8080%2C1.1732402573729 is not closed yet, will try archiving it next time
2024-11-23T22:56:13,749 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,749 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,749 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741835_1011 (size=93)
2024-11-23T22:56:13,749 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741835_1011 (size=93)
2024-11-23T22:56:13,750 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:13,750 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/WALs/test.com,8080,1/test.com%2C8080%2C1.1732402573729 to hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs/test.com%2C8080%2C1.1732402573729
2024-11-23T22:56:13,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741836_1012 (size=93)
2024-11-23T22:56:13,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741836_1012 (size=93)
2024-11-23T22:56:14,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs
2024-11-23T22:56:14,154 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732402573740)
2024-11-23T22:56:14,154 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-23T22:56:14,154 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:56:14,154 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:14,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:14,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:14,155 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-23T22:56:14,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-23T22:56:14,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2068951342, stopped=false
2024-11-23T22:56:14,155 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,35761,1732402571725
2024-11-23T22:56:14,198 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-23T22:56:14,199 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,202 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:14,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-23T22:56:14,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:14,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:14,205 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:56:14,205 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-23T22:56:14,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:14,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:14,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:14,205 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,34101,1732402571990' *****
2024-11-23T22:56:14,206 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-23T22:56:14,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-23T22:56:14,206 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:34101.
2024-11-23T22:56:14,206 DEBUG [RS:0;dd96e8dc8369:34101 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-23T22:56:14,206 DEBUG [RS:0;dd96e8dc8369:34101 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-23T22:56:14,206 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-23T22:56:14,207 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-23T22:56:14,207 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-23T22:56:14,207 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-23T22:56:14,207 DEBUG [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-23T22:56:14,207 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-23T22:56:14,207 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-23T22:56:14,207 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-23T22:56:14,207 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-23T22:56:14,207 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-23T22:56:14,207 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-23T22:56:14,223 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/.tmp/ns/1b433eb5dea24dba941b8bc6d3035cfb is 43, key is default/ns:d/1732402573682/Put/seqid=0
2024-11-23T22:56:14,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741837_1013 (size=5153)
2024-11-23T22:56:14,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741837_1013 (size=5153)
2024-11-23T22:56:14,231 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/.tmp/ns/1b433eb5dea24dba941b8bc6d3035cfb
2024-11-23T22:56:14,233 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,233 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,233 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.1732402386431
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:14,242 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,242 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,243 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,245 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/.tmp/ns/1b433eb5dea24dba941b8bc6d3035cfb as hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/ns/1b433eb5dea24dba941b8bc6d3035cfb
2024-11-23T22:56:14,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-23T22:56:14,251 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/ns/1b433eb5dea24dba941b8bc6d3035cfb, entries=2, sequenceid=6, filesize=5.0 K
2024-11-23T22:56:14,252 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false
2024-11-23T22:56:14,252 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-23T22:56:14,256 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-23T22:56:14,257 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-23T22:56:14,257 INFO  [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-23T22:56:14,257 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732402574207Running coprocessor pre-close hooks at 1732402574207Disabling compacts and flushes for region at 1732402574207Disabling writes for close at 1732402574207Obtaining lock to block concurrent updates at 1732402574207Preparing flush snapshotting stores in 1588230740 at 1732402574207Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732402574207Flushing stores of hbase:meta,,1.1588230740 at 1732402574208 (+1 ms)Flushing 1588230740/ns: creating writer at 1732402574208Flushing 1588230740/ns: appending metadata at 1732402574222 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732402574222Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4654fcae: reopening flushed file at 1732402574245 (+23 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false at 1732402574252 (+7 ms)Writing region close event to WAL at 1732402574253 (+1 ms)Running coprocessor post-close hooks at 1732402574257 (+4 ms)Closed at 1732402574257
2024-11-23T22:56:14,257 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-23T22:56:14,407 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,34101,1732402571990; all regions closed.
2024-11-23T22:56:14,407 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,408 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,408 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,408 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,408 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741834_1010 (size=1152)
2024-11-23T22:56:14,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741834_1010 (size=1152)
2024-11-23T22:56:14,411 DEBUG [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs
2024-11-23T22:56:14,411 INFO  [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C34101%2C1732402571990.meta:.meta(num 1732402573599)
2024-11-23T22:56:14,412 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,412 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,412 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,412 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,412 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741833_1009 (size=93)
2024-11-23T22:56:14,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741833_1009 (size=93)
2024-11-23T22:56:14,416 DEBUG [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/oldWALs
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C34101%2C1732402571990:(num 1732402573214)
2024-11-23T22:56:14,416 DEBUG [RS:0;dd96e8dc8369:34101 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.LeaseManager(133): Closed leases
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:56:14,416 INFO  [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:56:14,416 INFO  [RS:0;dd96e8dc8369:34101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34101
2024-11-23T22:56:14,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,34101,1732402571990
2024-11-23T22:56:14,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-23T22:56:14,429 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:56:14,439 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,34101,1732402571990]
2024-11-23T22:56:14,450 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,34101,1732402571990 already deleted, retry=false
2024-11-23T22:56:14,450 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,34101,1732402571990 expired; onlineServers=0
2024-11-23T22:56:14,450 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,35761,1732402571725' *****
2024-11-23T22:56:14,450 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-23T22:56:14,450 DEBUG [M:0;dd96e8dc8369:35761 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-23T22:56:14,450 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-23T22:56:14,450 DEBUG [M:0;dd96e8dc8369:35761 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-23T22:56:14,450 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402572928 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402572928,5,FailOnTimeoutGroup]
2024-11-23T22:56:14,450 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402572928 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402572928,5,FailOnTimeoutGroup]
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-23T22:56:14,450 DEBUG [M:0;dd96e8dc8369:35761 {}] master.HMaster(1795): Stopping service threads
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-23T22:56:14,450 INFO  [M:0;dd96e8dc8369:35761 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-23T22:56:14,451 INFO  [M:0;dd96e8dc8369:35761 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-23T22:56:14,451 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-23T22:56:14,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-23T22:56:14,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-23T22:56:14,460 DEBUG [M:0;dd96e8dc8369:35761 {}] zookeeper.ZKUtil(347): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-23T22:56:14,460 WARN  [M:0;dd96e8dc8369:35761 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-23T22:56:14,461 INFO  [M:0;dd96e8dc8369:35761 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/.lastflushedseqids
2024-11-23T22:56:14,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741838_1014 (size=99)
2024-11-23T22:56:14,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741838_1014 (size=99)
2024-11-23T22:56:14,466 INFO  [M:0;dd96e8dc8369:35761 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-23T22:56:14,466 INFO  [M:0;dd96e8dc8369:35761 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-23T22:56:14,466 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-23T22:56:14,466 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:14,466 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:14,466 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-23T22:56:14,466 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:14,466 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-23T22:56:14,481 DEBUG [M:0;dd96e8dc8369:35761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15abb950b1b148f19f324d7de02e3ce9 is 82, key is hbase:meta,,1/info:regioninfo/1732402573641/Put/seqid=0
2024-11-23T22:56:14,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741839_1015 (size=5672)
2024-11-23T22:56:14,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741839_1015 (size=5672)
2024-11-23T22:56:14,485 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15abb950b1b148f19f324d7de02e3ce9
2024-11-23T22:56:14,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,36341,1732402387015/dd96e8dc8369%2C36341%2C1732402387015.1732402387251
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:14,503 DEBUG [M:0;dd96e8dc8369:35761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36eb026100f241d5810adf7f377fd33e is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732402573686/Put/seqid=0
2024-11-23T22:56:14,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741840_1016 (size=5275)
2024-11-23T22:56:14,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741840_1016 (size=5275)
2024-11-23T22:56:14,508 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36eb026100f241d5810adf7f377fd33e
2024-11-23T22:56:14,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39715/user/jenkins/test-data/b89ad22e-10ed-47ba-35b6-f31172f7193c/WALs/dd96e8dc8369,43859,1732402385760/dd96e8dc8369%2C43859%2C1732402385760.meta.1732402386791.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-23T22:56:14,526 DEBUG [M:0;dd96e8dc8369:35761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d35607ffae1e44f3b8f2abcd07b473f9 is 69, key is dd96e8dc8369,34101,1732402571990/rs:state/1732402573059/Put/seqid=0
2024-11-23T22:56:14,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741841_1017 (size=5156)
2024-11-23T22:56:14,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741841_1017 (size=5156)
2024-11-23T22:56:14,531 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d35607ffae1e44f3b8f2abcd07b473f9
2024-11-23T22:56:14,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:14,539 INFO  [RS:0;dd96e8dc8369:34101 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:56:14,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34101-0x10169e965600001, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:14,539 INFO  [RS:0;dd96e8dc8369:34101 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,34101,1732402571990; zookeeper connection closed.
2024-11-23T22:56:14,546 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49cb9595 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49cb9595
2024-11-23T22:56:14,546 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-23T22:56:14,550 DEBUG [M:0;dd96e8dc8369:35761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d85bbb274bbb47499c075378355132c5 is 52, key is load_balancer_on/state:d/1732402573726/Put/seqid=0
2024-11-23T22:56:14,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741842_1018 (size=5056)
2024-11-23T22:56:14,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741842_1018 (size=5056)
2024-11-23T22:56:14,554 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d85bbb274bbb47499c075378355132c5
2024-11-23T22:56:14,558 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15abb950b1b148f19f324d7de02e3ce9 as hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/15abb950b1b148f19f324d7de02e3ce9
2024-11-23T22:56:14,562 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/15abb950b1b148f19f324d7de02e3ce9, entries=8, sequenceid=29, filesize=5.5 K
2024-11-23T22:56:14,563 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36eb026100f241d5810adf7f377fd33e as hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36eb026100f241d5810adf7f377fd33e
2024-11-23T22:56:14,567 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36eb026100f241d5810adf7f377fd33e, entries=3, sequenceid=29, filesize=5.2 K
2024-11-23T22:56:14,568 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d35607ffae1e44f3b8f2abcd07b473f9 as hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d35607ffae1e44f3b8f2abcd07b473f9
2024-11-23T22:56:14,572 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d35607ffae1e44f3b8f2abcd07b473f9, entries=1, sequenceid=29, filesize=5.0 K
2024-11-23T22:56:14,573 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d85bbb274bbb47499c075378355132c5 as hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d85bbb274bbb47499c075378355132c5
2024-11-23T22:56:14,577 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46691/user/jenkins/test-data/3652a927-60d1-89b7-33c8-2a19961610eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d85bbb274bbb47499c075378355132c5, entries=1, sequenceid=29, filesize=4.9 K
2024-11-23T22:56:14,578 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false
2024-11-23T22:56:14,580 INFO  [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-23T22:56:14,580 DEBUG [M:0;dd96e8dc8369:35761 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732402574466Disabling compacts and flushes for region at 1732402574466Disabling writes for close at 1732402574466Obtaining lock to block concurrent updates at 1732402574466Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402574466Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732402574467 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402574467Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402574467Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402574481 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402574481Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402574489 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402574502 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402574502Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402574512 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402574526 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402574526Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732402574534 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732402574549 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732402574549Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ac703a5: reopening flushed file at 1732402574558 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54508393: reopening flushed file at 1732402574562 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49473e51: reopening flushed file at 1732402574567 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0de37: reopening flushed file at 1732402574572 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false at 1732402574578 (+6 ms)Writing region close event to WAL at 1732402574579 (+1 ms)Closed at 1732402574579
2024-11-23T22:56:14,580 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,580 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,580 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,580 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,580 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-23T22:56:14,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36735 is added to blk_1073741830_1006 (size=10311)
2024-11-23T22:56:14,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40607 is added to blk_1073741830_1006 (size=10311)
2024-11-23T22:56:14,583 INFO  [M:0;dd96e8dc8369:35761 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-23T22:56:14,583 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-23T22:56:14,583 INFO  [M:0;dd96e8dc8369:35761 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35761
2024-11-23T22:56:14,583 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-23T22:56:14,694 INFO  [M:0;dd96e8dc8369:35761 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-23T22:56:14,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:14,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35761-0x10169e965600000, quorum=127.0.0.1:59848, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-23T22:56:14,697 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11e68b55{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:14,698 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@740bac32{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:14,698 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:14,698 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d140981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:14,698 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2074d4e0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:14,700 WARN  [BP-2361242-172.17.0.2-1732402569254 heartbeating to localhost/127.0.0.1:46691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:56:14,700 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:56:14,700 WARN  [BP-2361242-172.17.0.2-1732402569254 heartbeating to localhost/127.0.0.1:46691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2361242-172.17.0.2-1732402569254 (Datanode Uuid 04ea62da-9eed-41b4-9a91-441b2c672e61) service to localhost/127.0.0.1:46691
2024-11-23T22:56:14,700 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:56:14,701 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data3/current/BP-2361242-172.17.0.2-1732402569254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:14,701 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data4/current/BP-2361242-172.17.0.2-1732402569254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:14,701 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:56:14,703 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c04d24a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-23T22:56:14,703 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@550c39ff{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:14,703 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:14,704 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5139465a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:14,704 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29692e24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:14,705 WARN  [BP-2361242-172.17.0.2-1732402569254 heartbeating to localhost/127.0.0.1:46691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-23T22:56:14,705 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-23T22:56:14,705 WARN  [BP-2361242-172.17.0.2-1732402569254 heartbeating to localhost/127.0.0.1:46691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2361242-172.17.0.2-1732402569254 (Datanode Uuid b876a378-4509-46aa-b0cb-73a66058756b) service to localhost/127.0.0.1:46691
2024-11-23T22:56:14,705 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-23T22:56:14,705 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data1/current/BP-2361242-172.17.0.2-1732402569254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:14,706 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/cluster_30b4ee9e-f529-baf0-92ae-54aa5ae9061d/data/data2/current/BP-2361242-172.17.0.2-1732402569254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-23T22:56:14,706 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-23T22:56:14,712 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39cd0eab{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-23T22:56:14,712 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4833305a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-23T22:56:14,712 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-23T22:56:14,713 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6833f9d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-23T22:56:14,713 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ee84893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d49f4716-f0cb-0680-dc16-c5ab6dd254c2/hadoop.log.dir/,STOPPED}
2024-11-23T22:56:14,719 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-23T22:56:14,736 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-23T22:56:14,745 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=272 (was 233)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:46691 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:46691 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:46691 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46691
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:46691
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=6255 (was 6288)