2024-12-10 15:29:46,069 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-10 15:29:46,085 main DEBUG Took 0.013701 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-10 15:29:46,086 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-10 15:29:46,087 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-10 15:29:46,088 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-10 15:29:46,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,100 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-10 15:29:46,117 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,120 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,121 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,123 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,124 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,125 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,126 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,127 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,128 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,129 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,130 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,131 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 15:29:46,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,132 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-10 15:29:46,133 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 15:29:46,134 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-10 15:29:46,136 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-10 15:29:46,137 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-10 15:29:46,138 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-10 15:29:46,138 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-10 15:29:46,146 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-10 15:29:46,148 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-10 15:29:46,150 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-10 15:29:46,150 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-10 15:29:46,151 main DEBUG createAppenders(={Console})
2024-12-10 15:29:46,152 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-10 15:29:46,152 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-10 15:29:46,152 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-10 15:29:46,153 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-10 15:29:46,153 main DEBUG OutputStream closed
2024-12-10 15:29:46,154 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-10 15:29:46,154 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-10 15:29:46,154 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-10 15:29:46,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-10 15:29:46,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-10 15:29:46,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-10 15:29:46,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-10 15:29:46,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-10 15:29:46,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-10 15:29:46,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-10 15:29:46,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-10 15:29:46,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-10 15:29:46,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-10 15:29:46,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-10 15:29:46,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-10 15:29:46,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-10 15:29:46,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-10 15:29:46,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-10 15:29:46,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-10 15:29:46,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-10 15:29:46,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-10 15:29:46,255 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10 15:29:46,256 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-10 15:29:46,256 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-10 15:29:46,257 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-10T15:29:46,596 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075
2024-12-10 15:29:46,600 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-10 15:29:46,600 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10T15:29:46,611 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-10T15:29:46,655 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=5521
2024-12-10T15:29:46,657 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:29:46,679 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0, deleteOnExit=true
2024-12-10T15:29:46,680 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:29:46,681 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/test.cache.data in system properties and HBase conf
2024-12-10T15:29:46,682 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:29:46,684 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:29:46,685 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:29:46,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:29:46,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:29:46,798 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-10T15:29:46,942 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:29:46,948 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:29:46,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:29:46,949 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:29:46,950 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:29:46,951 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:29:46,951 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:29:46,952 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:29:46,953 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:29:46,954 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:29:46,954 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:29:46,955 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:29:46,956 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:29:46,956 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:29:46,957 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:29:47,543 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:29:48,171 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-10T15:29:48,236 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2826ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-10T15:29:48,296 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:29:48,344 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:29:48,359 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:29:48,360 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:29:48,361 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:29:48,371 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:29:48,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c5202f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:29:48,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ae7f863{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:29:48,534 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c1a236c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/java.io.tmpdir/jetty-localhost-40581-hadoop-hdfs-3_4_1-tests_jar-_-any-9416412042510260265/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:29:48,540 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a625720{HTTP/1.1, (http/1.1)}{localhost:40581}
2024-12-10T15:29:48,541 INFO  [Time-limited test {}] server.Server(415): Started @3131ms
2024-12-10T15:29:48,567 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:29:49,063 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:29:49,070 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:29:49,071 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:29:49,071 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:29:49,071 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:29:49,072 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@460757e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:29:49,073 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e911877{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:29:49,168 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b32dfc5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/java.io.tmpdir/jetty-localhost-41029-hadoop-hdfs-3_4_1-tests_jar-_-any-7312096236186113089/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:29:49,168 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43ebd249{HTTP/1.1, (http/1.1)}{localhost:41029}
2024-12-10T15:29:49,169 INFO  [Time-limited test {}] server.Server(415): Started @3759ms
2024-12-10T15:29:49,216 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:29:49,311 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:29:49,315 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:29:49,317 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:29:49,317 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:29:49,317 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:29:49,318 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@154eec55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:29:49,318 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29058af{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:29:49,416 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@78ec6c63{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/java.io.tmpdir/jetty-localhost-36293-hadoop-hdfs-3_4_1-tests_jar-_-any-2773233881452796165/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:29:49,417 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@633713c3{HTTP/1.1, (http/1.1)}{localhost:36293}
2024-12-10T15:29:49,417 INFO  [Time-limited test {}] server.Server(415): Started @4008ms
2024-12-10T15:29:49,419 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:29:50,569 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data2/current/BP-2125556533-172.17.0.2-1733844587642/current, will proceed with Du for space computation calculation, 
2024-12-10T15:29:50,569 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data1/current/BP-2125556533-172.17.0.2-1733844587642/current, will proceed with Du for space computation calculation, 
2024-12-10T15:29:50,569 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data3/current/BP-2125556533-172.17.0.2-1733844587642/current, will proceed with Du for space computation calculation, 
2024-12-10T15:29:50,569 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data4/current/BP-2125556533-172.17.0.2-1733844587642/current, will proceed with Du for space computation calculation, 
2024-12-10T15:29:50,600 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:29:50,600 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:29:50,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x657c1570d30bdbc9 with lease ID 0x4206805fba02bc7b: Processing first storage report for DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9 from datanode DatanodeRegistration(127.0.0.1:34063, datanodeUuid=30f110e1-e152-4654-bb26-4478652fcebc, infoPort=41823, infoSecurePort=0, ipcPort=40775, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642)
2024-12-10T15:29:50,644 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x657c1570d30bdbc9 with lease ID 0x4206805fba02bc7b: from storage DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9 node DatanodeRegistration(127.0.0.1:34063, datanodeUuid=30f110e1-e152-4654-bb26-4478652fcebc, infoPort=41823, infoSecurePort=0, ipcPort=40775, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:29:50,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3181a808ca3e5fb7 with lease ID 0x4206805fba02bc7c: Processing first storage report for DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9 from datanode DatanodeRegistration(127.0.0.1:32771, datanodeUuid=64cd6da1-1fc7-44b2-917a-b934dbae9e3f, infoPort=34361, infoSecurePort=0, ipcPort=39139, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642)
2024-12-10T15:29:50,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3181a808ca3e5fb7 with lease ID 0x4206805fba02bc7c: from storage DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9 node DatanodeRegistration(127.0.0.1:32771, datanodeUuid=64cd6da1-1fc7-44b2-917a-b934dbae9e3f, infoPort=34361, infoSecurePort=0, ipcPort=39139, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:29:50,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x657c1570d30bdbc9 with lease ID 0x4206805fba02bc7b: Processing first storage report for DS-7aa11cb4-90e3-4d84-817c-66f9fc766e4a from datanode DatanodeRegistration(127.0.0.1:34063, datanodeUuid=30f110e1-e152-4654-bb26-4478652fcebc, infoPort=41823, infoSecurePort=0, ipcPort=40775, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642)
2024-12-10T15:29:50,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x657c1570d30bdbc9 with lease ID 0x4206805fba02bc7b: from storage DS-7aa11cb4-90e3-4d84-817c-66f9fc766e4a node DatanodeRegistration(127.0.0.1:34063, datanodeUuid=30f110e1-e152-4654-bb26-4478652fcebc, infoPort=41823, infoSecurePort=0, ipcPort=40775, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:29:50,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3181a808ca3e5fb7 with lease ID 0x4206805fba02bc7c: Processing first storage report for DS-b687eb8a-1235-4d7d-9309-93a7dd738082 from datanode DatanodeRegistration(127.0.0.1:32771, datanodeUuid=64cd6da1-1fc7-44b2-917a-b934dbae9e3f, infoPort=34361, infoSecurePort=0, ipcPort=39139, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642)
2024-12-10T15:29:50,646 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3181a808ca3e5fb7 with lease ID 0x4206805fba02bc7c: from storage DS-b687eb8a-1235-4d7d-9309-93a7dd738082 node DatanodeRegistration(127.0.0.1:32771, datanodeUuid=64cd6da1-1fc7-44b2-917a-b934dbae9e3f, infoPort=34361, infoSecurePort=0, ipcPort=39139, storageInfo=lv=-57;cid=testClusterID;nsid=666834822;c=1733844587642), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:29:50,718 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075
2024-12-10T15:29:50,784 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/zookeeper_0, clientPort=55710, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:29:50,792 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55710
2024-12-10T15:29:50,805 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:50,809 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:51,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:29:51,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:29:51,414 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a with version=8
2024-12-10T15:29:51,414 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:29:51,485 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-10T15:29:51,748 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:29:51,758 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:51,759 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:51,764 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:29:51,764 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:51,765 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:29:51,921 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:29:51,992 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-10T15:29:52,001 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-10T15:29:52,005 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:29:52,032 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 60545 (auto-detected)
2024-12-10T15:29:52,033 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-10T15:29:52,053 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33141
2024-12-10T15:29:52,079 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33141 connecting to ZooKeeper ensemble=127.0.0.1:55710
2024-12-10T15:29:52,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331410x0, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:29:52,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33141-0x1001058e7790000 connected
2024-12-10T15:29:52,323 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:52,327 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:52,337 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:29:52,341 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a, hbase.cluster.distributed=false
2024-12-10T15:29:52,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:29:52,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33141
2024-12-10T15:29:52,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33141
2024-12-10T15:29:52,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33141
2024-12-10T15:29:52,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33141
2024-12-10T15:29:52,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33141
2024-12-10T15:29:52,462 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:29:52,463 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:52,464 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:52,464 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:29:52,464 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:29:52,464 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:29:52,466 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:29:52,468 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:29:52,469 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45265
2024-12-10T15:29:52,471 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45265 connecting to ZooKeeper ensemble=127.0.0.1:55710
2024-12-10T15:29:52,472 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:52,474 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:52,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452650x0, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:29:52,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452650x0, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:29:52,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45265-0x1001058e7790001 connected
2024-12-10T15:29:52,495 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:29:52,503 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:29:52,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:29:52,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:29:52,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45265
2024-12-10T15:29:52,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45265
2024-12-10T15:29:52,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45265
2024-12-10T15:29:52,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45265
2024-12-10T15:29:52,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45265
2024-12-10T15:29:52,526 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:33141
2024-12-10T15:29:52,527 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,33141,1733844591575
2024-12-10T15:29:52,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:29:52,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:29:52,544 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,33141,1733844591575
2024-12-10T15:29:52,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:52,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:29:52,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:52,574 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:29:52,575 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,33141,1733844591575 from backup master directory
2024-12-10T15:29:52,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,33141,1733844591575
2024-12-10T15:29:52,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:29:52,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:29:52,584 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:29:52,584 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,33141,1733844591575
2024-12-10T15:29:52,586 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-10T15:29:52,587 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-10T15:29:52,638 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase.id] with ID: b8c4504b-684d-45c5-8ee5-435ddb8dd345
2024-12-10T15:29:52,638 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/.tmp/hbase.id
2024-12-10T15:29:52,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:29:52,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:29:52,649 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/.tmp/hbase.id]:[hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase.id]
2024-12-10T15:29:52,693 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:52,698 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:29:52,715 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms.
2024-12-10T15:29:52,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:52,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:52,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:29:52,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:29:52,764 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:29:52,766 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:29:52,770 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:29:52,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:29:52,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:29:52,813 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store
2024-12-10T15:29:52,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:29:52,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:29:52,835 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-10T15:29:52,838 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:52,839 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:29:52,840 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:29:52,840 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:29:52,841 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:29:52,841 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:29:52,841 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:29:52,842 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844592839Disabling compacts and flushes for region at 1733844592839Disabling writes for close at 1733844592841 (+2 ms)Writing region close event to WAL at 1733844592841Closed at 1733844592841
2024-12-10T15:29:52,844 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/.initializing
2024-12-10T15:29:52,844 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/WALs/d6bcd7308055,33141,1733844591575
2024-12-10T15:29:52,864 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C33141%2C1733844591575, suffix=, logDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/WALs/d6bcd7308055,33141,1733844591575, archiveDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/oldWALs, maxLogs=10
2024-12-10T15:29:52,872 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C33141%2C1733844591575.1733844592868
2024-12-10T15:29:52,893 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/WALs/d6bcd7308055,33141,1733844591575/d6bcd7308055%2C33141%2C1733844591575.1733844592868
2024-12-10T15:29:52,906 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:29:52,907 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:29:52,907 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:52,910 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,911 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,945 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:29:52,966 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:29:52,969 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:52,971 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:52,972 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:29:52,975 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:29:52,975 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:52,976 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:29:52,976 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:29:52,979 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:29:52,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:52,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:29:52,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:29:52,983 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:29:52,983 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:52,984 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:29:52,985 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,988 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,989 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,996 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:52,997 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:53,002 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:29:53,008 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:29:53,013 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:29:53,014 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=710757, jitterRate=-0.09622615575790405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:29:53,020 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844592922Initializing all the Stores at 1733844592924 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844592925 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844592926 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844592926Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844592926Cleaning up temporary data from old regions at 1733844592997 (+71 ms)Region opened successfully at 1733844593020 (+23 ms)
2024-12-10T15:29:53,022 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:29:53,054 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2830a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:29:53,079 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:29:53,089 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:29:53,089 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:29:53,093 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:29:53,095 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-10T15:29:53,100 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-10T15:29:53,100 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:29:53,122 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:29:53,129 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:29:53,141 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:29:53,144 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:29:53,146 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:29:53,151 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:29:53,155 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:29:53,159 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:29:53,172 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:29:53,175 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:29:53,183 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:29:53,205 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:29:53,214 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:29:53,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:29:53,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:29:53,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,230 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,33141,1733844591575, sessionid=0x1001058e7790000, setting cluster-up flag (Was=false)
2024-12-10T15:29:53,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,288 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:29:53,291 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,33141,1733844591575
2024-12-10T15:29:53,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:53,341 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:29:53,346 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,33141,1733844591575
2024-12-10T15:29:53,355 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:29:53,416 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(746): ClusterId : b8c4504b-684d-45c5-8ee5-435ddb8dd345
2024-12-10T15:29:53,419 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:29:53,419 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:29:53,427 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:29:53,427 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:29:53,428 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:29:53,434 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:29:53,437 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:29:53,438 DEBUG [RS:0;d6bcd7308055:45265 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2744ff68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:29:53,442 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,33141,1733844591575
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:29:53,450 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:29:53,451 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:45265
2024-12-10T15:29:53,451 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:29:53,451 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:29:53,451 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:29:53,451 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:29:53,452 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,452 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:29:53,452 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,453 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:29:53,454 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:29:53,454 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:29:53,457 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,33141,1733844591575 with port=45265, startcode=1733844592430
2024-12-10T15:29:53,457 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:29:53,457 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:29:53,459 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844623459
2024-12-10T15:29:53,461 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:29:53,462 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:29:53,463 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:53,463 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:29:53,466 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:29:53,466 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:29:53,467 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:29:53,467 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:29:53,468 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,471 DEBUG [RS:0;d6bcd7308055:45265 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:29:53,473 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:29:53,474 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:29:53,475 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:29:53,478 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:29:53,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:29:53,480 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:29:53,481 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a
2024-12-10T15:29:53,481 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:29:53,481 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:29:53,483 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844593482,5,FailOnTimeoutGroup]
2024-12-10T15:29:53,484 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844593483,5,FailOnTimeoutGroup]
2024-12-10T15:29:53,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:29:53,486 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,486 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:29:53,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:29:53,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:53,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:29:53,506 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:29:53,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:53,507 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:53,507 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:29:53,510 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:29:53,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:53,511 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:53,512 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:29:53,515 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:29:53,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:53,516 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:53,517 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:29:53,521 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:29:53,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:53,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:53,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:29:53,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740
2024-12-10T15:29:53,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740
2024-12-10T15:29:53,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:29:53,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:29:53,531 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:29:53,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:29:53,538 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:29:53,539 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702799, jitterRate=-0.10634574294090271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:29:53,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844593501Initializing all the Stores at 1733844593503 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844593503Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844593503Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844593503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844593503Cleaning up temporary data from old regions at 1733844593529 (+26 ms)Region opened successfully at 1733844593541 (+12 ms)
2024-12-10T15:29:53,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:29:53,542 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:29:53,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:29:53,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:29:53,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:29:53,544 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:29:53,544 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56843, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:29:53,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844593542Disabling compacts and flushes for region at 1733844593542Disabling writes for close at 1733844593543 (+1 ms)Writing region close event to WAL at 1733844593544 (+1 ms)Closed at 1733844593544
2024-12-10T15:29:53,548 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:29:53,548 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:29:53,550 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33141 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,552 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33141 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,554 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:29:53,561 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:29:53,563 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:29:53,565 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a
2024-12-10T15:29:53,565 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44521
2024-12-10T15:29:53,565 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:29:53,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:29:53,573 DEBUG [RS:0;d6bcd7308055:45265 {}] zookeeper.ZKUtil(111): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,573 WARN  [RS:0;d6bcd7308055:45265 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:29:53,573 INFO  [RS:0;d6bcd7308055:45265 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:29:53,574 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,575 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,45265,1733844592430]
2024-12-10T15:29:53,596 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:29:53,609 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:29:53,614 INFO  [RS:0;d6bcd7308055:45265 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:29:53,615 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,616 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:29:53,622 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:29:53,623 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,623 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,623 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,624 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,624 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,624 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,624 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,625 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:29:53,626 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:29:53,626 DEBUG [RS:0;d6bcd7308055:45265 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:29:53,627 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,627 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,627 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,628 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,628 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,628 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,45265,1733844592430-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:29:53,643 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:29:53,645 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,45265,1733844592430-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,645 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,645 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.Replication(171): d6bcd7308055,45265,1733844592430 started
2024-12-10T15:29:53,660 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:53,660 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,45265,1733844592430, RpcServer on d6bcd7308055/172.17.0.2:45265, sessionid=0x1001058e7790001
2024-12-10T15:29:53,661 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:29:53,661 DEBUG [RS:0;d6bcd7308055:45265 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,661 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,45265,1733844592430'
2024-12-10T15:29:53,661 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:29:53,662 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:29:53,663 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:29:53,663 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:29:53,663 DEBUG [RS:0;d6bcd7308055:45265 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,663 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,45265,1733844592430'
2024-12-10T15:29:53,663 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:29:53,664 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:29:53,664 DEBUG [RS:0;d6bcd7308055:45265 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:29:53,665 INFO  [RS:0;d6bcd7308055:45265 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:29:53,665 INFO  [RS:0;d6bcd7308055:45265 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:29:53,714 WARN  [d6bcd7308055:33141 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:29:53,780 INFO  [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C45265%2C1733844592430, suffix=, logDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430, archiveDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs, maxLogs=32
2024-12-10T15:29:53,782 INFO  [RS:0;d6bcd7308055:45265 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844593782
2024-12-10T15:29:53,790 INFO  [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844593782
2024-12-10T15:29:53,792 DEBUG [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:29:53,969 DEBUG [d6bcd7308055:33141 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:29:53,981 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:53,987 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,45265,1733844592430, state=OPENING
2024-12-10T15:29:54,036 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:29:54,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:54,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:29:54,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:29:54,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:29:54,052 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:29:54,055 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,45265,1733844592430}]
2024-12-10T15:29:54,235 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:29:54,238 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52189, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:29:54,248 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:29:54,248 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:29:54,251 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C45265%2C1733844592430.meta, suffix=.meta, logDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430, archiveDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs, maxLogs=32
2024-12-10T15:29:54,253 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.meta.1733844594253.meta
2024-12-10T15:29:54,260 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.meta.1733844594253.meta
2024-12-10T15:29:54,262 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34361:34361),(127.0.0.1/127.0.0.1:41823:41823)]
2024-12-10T15:29:54,265 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:29:54,267 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:29:54,269 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:29:54,273 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:29:54,277 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:29:54,278 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:54,278 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:29:54,278 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:29:54,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:29:54,283 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:29:54,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:54,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:54,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:29:54,286 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:29:54,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:54,287 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:54,287 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:29:54,288 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:29:54,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:54,289 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:54,289 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:29:54,291 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:29:54,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:54,291 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:29:54,292 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:29:54,294 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740
2024-12-10T15:29:54,296 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740
2024-12-10T15:29:54,299 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:29:54,299 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:29:54,300 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:29:54,304 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:29:54,306 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811686, jitterRate=0.03211241960525513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:29:54,306 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:29:54,308 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844594279Writing region info on filesystem at 1733844594279Initializing all the Stores at 1733844594281 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844594281Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844594281Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844594281Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844594281Cleaning up temporary data from old regions at 1733844594299 (+18 ms)Running coprocessor post-open hooks at 1733844594306 (+7 ms)Region opened successfully at 1733844594307 (+1 ms)
2024-12-10T15:29:54,314 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844594226
2024-12-10T15:29:54,325 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:29:54,326 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:29:54,328 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:54,330 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,45265,1733844592430, state=OPEN
2024-12-10T15:29:54,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:29:54,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:29:54,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:29:54,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:29:54,374 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:54,381 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:29:54,381 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,45265,1733844592430 in 320 msec
2024-12-10T15:29:54,388 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:29:54,388 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 829 msec
2024-12-10T15:29:54,389 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:29:54,389 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:29:54,408 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:29:54,409 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,45265,1733844592430, seqNum=-1]
2024-12-10T15:29:54,427 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:29:54,431 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:29:54,456 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0730 sec
2024-12-10T15:29:54,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844594457, completionTime=-1
2024-12-10T15:29:54,459 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:29:54,459 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:29:54,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:29:54,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844654484
2024-12-10T15:29:54,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844714484
2024-12-10T15:29:54,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec
2024-12-10T15:29:54,488 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,488 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,488 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,490 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:33141, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,490 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,491 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,496 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:29:54,524 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.939sec
2024-12-10T15:29:54,525 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:29:54,527 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:29:54,528 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:29:54,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:29:54,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:29:54,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:29:54,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:29:54,539 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:29:54,541 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:29:54,542 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,33141,1733844591575-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:29:54,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43835ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:29:54,631 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-10T15:29:54,632 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-10T15:29:54,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,33141,-1 for getting cluster id
2024-12-10T15:29:54,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:29:54,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b8c4504b-684d-45c5-8ee5-435ddb8dd345'
2024-12-10T15:29:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:29:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b8c4504b-684d-45c5-8ee5-435ddb8dd345"

2024-12-10T15:29:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50031a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:29:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,33141,-1]
2024-12-10T15:29:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:29:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:29:54,663 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:29:54,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d9508a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:29:54,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:29:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,45265,1733844592430, seqNum=-1]
2024-12-10T15:29:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:29:54,676 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:29:54,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,33141,1733844591575
2024-12-10T15:29:54,699 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:29:54,706 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:29:54,711 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T15:29:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d6bcd7308055,33141,1733844591575
2024-12-10T15:29:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53adb2ba
2024-12-10T15:29:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T15:29:54,747 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T15:29:54,750 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T15:29:54,750 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T15:29:54,754 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:29:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-10T15:29:54,765 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T15:29:54,768 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-10T15:29:54,768 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:54,771 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T15:29:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:29:54,825 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741835_1011 (size=389)
2024-12-10T15:29:54,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741835_1011 (size=389)
2024-12-10T15:29:54,830 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3eb3a6d6d7ebe91c66d296faa7fd4edc, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a
2024-12-10T15:29:54,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741836_1012 (size=72)
2024-12-10T15:29:54,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741836_1012 (size=72)
2024-12-10T15:29:54,846 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:54,847 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 3eb3a6d6d7ebe91c66d296faa7fd4edc, disabling compactions & flushes
2024-12-10T15:29:54,847 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:54,847 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:54,847 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc. after waiting 0 ms
2024-12-10T15:29:54,847 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:54,847 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:54,847 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:
Waiting for close lock at 1733844594846Disabling compacts and flushes for region at 1733844594846Disabling writes for close at 1733844594847 (+1 ms)Writing region close event to WAL at 1733844594847Closed at 1733844594847
2024-12-10T15:29:54,850 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T15:29:54,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733844594850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844594850"}]},"ts":"1733844594850"}
2024-12-10T15:29:54,861 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T15:29:54,863 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T15:29:54,866 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844594863"}]},"ts":"1733844594863"}
2024-12-10T15:29:54,870 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-10T15:29:54,872 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3eb3a6d6d7ebe91c66d296faa7fd4edc, ASSIGN}]
2024-12-10T15:29:54,875 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3eb3a6d6d7ebe91c66d296faa7fd4edc, ASSIGN
2024-12-10T15:29:54,878 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3eb3a6d6d7ebe91c66d296faa7fd4edc, ASSIGN; state=OFFLINE, location=d6bcd7308055,45265,1733844592430; forceNewPlan=false, retain=false
2024-12-10T15:29:55,029 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3eb3a6d6d7ebe91c66d296faa7fd4edc, regionState=OPENING, regionLocation=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:55,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3eb3a6d6d7ebe91c66d296faa7fd4edc, ASSIGN because future has completed
2024-12-10T15:29:55,036 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eb3a6d6d7ebe91c66d296faa7fd4edc, server=d6bcd7308055,45265,1733844592430}]
2024-12-10T15:29:55,197 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:55,198 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eb3a6d6d7ebe91c66d296faa7fd4edc, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:29:55,199 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,199 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:29:55,199 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,200 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,203 INFO  [StoreOpener-3eb3a6d6d7ebe91c66d296faa7fd4edc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3eb3a6d6d7ebe91c66d296faa7fd4edc 
2024-12-10T15:29:55,206 INFO  [StoreOpener-3eb3a6d6d7ebe91c66d296faa7fd4edc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eb3a6d6d7ebe91c66d296faa7fd4edc columnFamilyName info
2024-12-10T15:29:55,206 DEBUG [StoreOpener-3eb3a6d6d7ebe91c66d296faa7fd4edc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:29:55,208 INFO  [StoreOpener-3eb3a6d6d7ebe91c66d296faa7fd4edc-1 {}] regionserver.HStore(327): Store=3eb3a6d6d7ebe91c66d296faa7fd4edc/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:29:55,208 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,210 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,210 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,211 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,211 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,214 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,218 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:29:55,219 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3eb3a6d6d7ebe91c66d296faa7fd4edc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692096, jitterRate=-0.11995473504066467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:29:55,219 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:29:55,219 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:
Running coprocessor pre-open hook at 1733844595200Writing region info on filesystem at 1733844595200Initializing all the Stores at 1733844595202 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844595202Cleaning up temporary data from old regions at 1733844595211 (+9 ms)Running coprocessor post-open hooks at 1733844595219 (+8 ms)Region opened successfully at 1733844595219
2024-12-10T15:29:55,221 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc., pid=6, masterSystemTime=1733844595190
2024-12-10T15:29:55,225 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:55,225 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:29:55,226 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3eb3a6d6d7ebe91c66d296faa7fd4edc, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,45265,1733844592430
2024-12-10T15:29:55,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eb3a6d6d7ebe91c66d296faa7fd4edc, server=d6bcd7308055,45265,1733844592430 because future has completed
2024-12-10T15:29:55,235 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T15:29:55,236 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eb3a6d6d7ebe91c66d296faa7fd4edc, server=d6bcd7308055,45265,1733844592430 in 195 msec
2024-12-10T15:29:55,239 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T15:29:55,239 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3eb3a6d6d7ebe91c66d296faa7fd4edc, ASSIGN in 363 msec
2024-12-10T15:29:55,241 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T15:29:55,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844595241"}]},"ts":"1733844595241"}
2024-12-10T15:29:55,245 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-10T15:29:55,247 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T15:29:55,250 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 490 msec
2024-12-10T15:29:59,787 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-10T15:29:59,828 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T15:29:59,829 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-10T15:30:01,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:30:01,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T15:30:01,990 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-10T15:30:01,990 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T15:30:01,991 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:30:01,991 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T15:30:01,992 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T15:30:01,992 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-10T15:30:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:30:04,821 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-10T15:30:04,824 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-10T15:30:04,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-10T15:30:04,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:30:04,834 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844604834
2024-12-10T15:30:04,843 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:04,844 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:04,844 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:04,844 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:04,844 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:04,845 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844593782 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844604834
2024-12-10T15:30:04,851 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34361:34361),(127.0.0.1/127.0.0.1:41823:41823)]
2024-12-10T15:30:04,851 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844593782 is not closed yet, will try archiving it next time
2024-12-10T15:30:04,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741833_1009 (size=451)
2024-12-10T15:30:04,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741833_1009 (size=451)
2024-12-10T15:30:04,857 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844593782 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844593782
2024-12-10T15:30:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc., hostname=d6bcd7308055,45265,1733844592430, seqNum=2]
2024-12-10T15:30:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45265 {}] regionserver.HRegion(8855): Flush requested on 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:30:16,897 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:30:16,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/76539b9db7334d3893d123a254956603 is 1080, key is row0001/info:/1733844604864/Put/seqid=0
2024-12-10T15:30:16,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741838_1014 (size=12509)
2024-12-10T15:30:16,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741838_1014 (size=12509)
2024-12-10T15:30:16,973 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/76539b9db7334d3893d123a254956603
2024-12-10T15:30:17,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/76539b9db7334d3893d123a254956603 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603
2024-12-10T15:30:17,042 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603, entries=7, sequenceid=11, filesize=12.2 K
2024-12-10T15:30:17,050 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 152ms, sequenceid=11, compaction requested=false
2024-12-10T15:30:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:

2024-12-10T15:30:20,713 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:30:24,906 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844624905
2024-12-10T15:30:25,124 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 213 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:25,124 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:25,124 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:25,124 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:25,124 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:25,125 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:25,125 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844604834 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844624905
2024-12-10T15:30:25,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34361:34361),(127.0.0.1/127.0.0.1:41823:41823)]
2024-12-10T15:30:25,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844604834 is not closed yet, will try archiving it next time
2024-12-10T15:30:25,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741837_1013 (size=12399)
2024-12-10T15:30:25,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741837_1013 (size=12399)
2024-12-10T15:30:25,332 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:27,542 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:29,747 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:31,952 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45265 {}] regionserver.HRegion(8855): Flush requested on 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:30:31,952 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:30:32,155 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:32,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/5d09338089ac4027a1e9c5004dd4de86 is 1080, key is row0008/info:/1733844618895/Put/seqid=0
2024-12-10T15:30:32,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741840_1016 (size=12509)
2024-12-10T15:30:32,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741840_1016 (size=12509)
2024-12-10T15:30:32,176 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/5d09338089ac4027a1e9c5004dd4de86
2024-12-10T15:30:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/5d09338089ac4027a1e9c5004dd4de86 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86
2024-12-10T15:30:32,196 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86, entries=7, sequenceid=21, filesize=12.2 K
2024-12-10T15:30:32,399 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:32,400 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 447ms, sequenceid=21, compaction requested=false
2024-12-10T15:30:32,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:

2024-12-10T15:30:32,400 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-10T15:30:32,401 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:30:32,402 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603 because midkey is the same as first or last row
2024-12-10T15:30:34,159 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:34,748 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T15:30:34,748 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T15:30:36,367 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:36,373 WARN  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:36,375 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C45265%2C1733844592430:(num 1733844624905) roll requested
2024-12-10T15:30:36,376 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844636376
2024-12-10T15:30:36,585 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:36,585 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:36,586 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:36,586 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:36,586 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:36,586 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:36,586 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844624905 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844636376
2024-12-10T15:30:36,588 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:30:36,588 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844624905 is not closed yet, will try archiving it next time
2024-12-10T15:30:36,588 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844604834 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844604834
2024-12-10T15:30:36,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741839_1015 (size=7739)
2024-12-10T15:30:36,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741839_1015 (size=7739)
2024-12-10T15:30:38,572 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:40,199 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 3eb3a6d6d7ebe91c66d296faa7fd4edc, had cached 0 bytes from a total of 25018
2024-12-10T15:30:40,776 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:42,982 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:45,187 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:47,188 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T15:30:47,189 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844647189
2024-12-10T15:30:50,713 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:30:52,199 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:52,202 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:30:52,202 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C45265%2C1733844592430:(num 1733844647189) roll requested
2024-12-10T15:30:52,202 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:52,202 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:52,202 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:52,203 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:52,203 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:30:52,203 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844636376 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844647189
2024-12-10T15:30:52,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34361:34361),(127.0.0.1/127.0.0.1:41823:41823)]
2024-12-10T15:30:52,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844636376 is not closed yet, will try archiving it next time
2024-12-10T15:30:52,206 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844652206
2024-12-10T15:30:52,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741841_1017 (size=4753)
2024-12-10T15:30:52,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741841_1017 (size=4753)
2024-12-10T15:30:57,211 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:57,212 WARN  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45265 {}] regionserver.HRegion(8855): Flush requested on 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:30:57,212 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:30:57,220 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:57,220 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:30:59,213 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T15:31:02,216 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:31:02,216 WARN  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK], DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK]]
2024-12-10T15:31:02,216 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:02,217 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:02,217 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:02,217 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:02,218 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:02,218 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844647189 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844652206
2024-12-10T15:31:02,220 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:31:02,221 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844647189 is not closed yet, will try archiving it next time
2024-12-10T15:31:02,221 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C45265%2C1733844592430:(num 1733844652206) roll requested
2024-12-10T15:31:02,222 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844662221
2024-12-10T15:31:02,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741842_1018 (size=1569)
2024-12-10T15:31:02,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741842_1018 (size=1569)
2024-12-10T15:31:02,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/57e2fd9fe13d4c6b86da54693253a1d2 is 1080, key is row0015/info:/1733844633956/Put/seqid=0
2024-12-10T15:31:02,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741844_1020 (size=12509)
2024-12-10T15:31:02,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741844_1020 (size=12509)
2024-12-10T15:31:02,234 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/57e2fd9fe13d4c6b86da54693253a1d2
2024-12-10T15:31:02,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/57e2fd9fe13d4c6b86da54693253a1d2 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2
2024-12-10T15:31:02,255 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2, entries=7, sequenceid=31, filesize=12.2 K
2024-12-10T15:31:07,238 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5013 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:31:07,238 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5013 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:31:07,257 INFO  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:31:07,258 WARN  [FSHLog-0-hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a-prefix:d6bcd7308055,45265,1733844592430 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34063,DS-02445cd9-9800-4ee7-90fb-aa755f3b2bd9,DISK], DatanodeInfoWithStorage[127.0.0.1:32771,DS-7bbeaa36-4be8-4433-8a18-cb4808e69af9,DISK]]
2024-12-10T15:31:07,258 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 10045ms, sequenceid=31, compaction requested=true
2024-12-10T15:31:07,258 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:

2024-12-10T15:31:07,258 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-10T15:31:07,259 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:07,259 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603 because midkey is the same as first or last row
2024-12-10T15:31:07,259 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,259 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,260 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844652206 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844662221
2024-12-10T15:31:07,261 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34361:34361),(127.0.0.1/127.0.0.1:41823:41823)]
2024-12-10T15:31:07,261 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844652206 is not closed yet, will try archiving it next time
2024-12-10T15:31:07,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3eb3a6d6d7ebe91c66d296faa7fd4edc:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:31:07,261 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844624905 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844624905
2024-12-10T15:31:07,261 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C45265%2C1733844592430:(num 1733844667261) roll requested
2024-12-10T15:31:07,261 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844667261
2024-12-10T15:31:07,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741843_1019 (size=438)
2024-12-10T15:31:07,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:31:07,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741843_1019 (size=438)
2024-12-10T15:31:07,264 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844636376 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844636376
2024-12-10T15:31:07,264 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:31:07,265 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844647189 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844647189
2024-12-10T15:31:07,267 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844652206 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844652206
2024-12-10T15:31:07,268 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:31:07,269 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HStore(1541): 3eb3a6d6d7ebe91c66d296faa7fd4edc/info is initiating minor compaction (all files)
2024-12-10T15:31:07,270 INFO  [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3eb3a6d6d7ebe91c66d296faa7fd4edc/info in TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:07,270 INFO  [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2] into tmpdir=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp, totalSize=36.6 K
2024-12-10T15:31:07,271 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76539b9db7334d3893d123a254956603, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733844604864
2024-12-10T15:31:07,272 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d09338089ac4027a1e9c5004dd4de86, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733844618895
2024-12-10T15:31:07,272 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e2fd9fe13d4c6b86da54693253a1d2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733844633956
2024-12-10T15:31:07,274 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,274 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,275 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,275 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,275 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,275 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844662221 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844667261
2024-12-10T15:31:07,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:31:07,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844662221 is not closed yet, will try archiving it next time
2024-12-10T15:31:07,276 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C45265%2C1733844592430.1733844667276
2024-12-10T15:31:07,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741845_1021 (size=93)
2024-12-10T15:31:07,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741845_1021 (size=93)
2024-12-10T15:31:07,282 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844662221 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs/d6bcd7308055%2C45265%2C1733844592430.1733844662221
2024-12-10T15:31:07,291 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,291 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,291 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,291 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,292 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:07,292 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844667261 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/WALs/d6bcd7308055,45265,1733844592430/d6bcd7308055%2C45265%2C1733844592430.1733844667276
2024-12-10T15:31:07,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741846_1022 (size=1258)
2024-12-10T15:31:07,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741846_1022 (size=1258)
2024-12-10T15:31:07,295 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41823:41823),(127.0.0.1/127.0.0.1:34361:34361)]
2024-12-10T15:31:07,304 INFO  [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3eb3a6d6d7ebe91c66d296faa7fd4edc#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:31:07,306 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/1399045e5d4d4b4884f4e78b5601406b is 1080, key is row0001/info:/1733844604864/Put/seqid=0
2024-12-10T15:31:07,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741848_1024 (size=27710)
2024-12-10T15:31:07,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741848_1024 (size=27710)
2024-12-10T15:31:07,322 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/1399045e5d4d4b4884f4e78b5601406b as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/1399045e5d4d4b4884f4e78b5601406b
2024-12-10T15:31:07,340 INFO  [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3eb3a6d6d7ebe91c66d296faa7fd4edc/info of 3eb3a6d6d7ebe91c66d296faa7fd4edc into 1399045e5d4d4b4884f4e78b5601406b(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:31:07,340 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:

2024-12-10T15:31:07,342 INFO  [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc., storeName=3eb3a6d6d7ebe91c66d296faa7fd4edc/info, priority=13, startTime=1733844667261; duration=0sec
2024-12-10T15:31:07,342 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T15:31:07,342 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:07,342 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/1399045e5d4d4b4884f4e78b5601406b because midkey is the same as first or last row
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/1399045e5d4d4b4884f4e78b5601406b because midkey is the same as first or last row
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/1399045e5d4d4b4884f4e78b5601406b because midkey is the same as first or last row
2024-12-10T15:31:07,343 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:31:07,344 DEBUG [RS:0;d6bcd7308055:45265-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3eb3a6d6d7ebe91c66d296faa7fd4edc:info
2024-12-10T15:31:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45265 {}] regionserver.HRegion(8855): Flush requested on 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:31:19,312 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:31:19,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/e618f54eecea4bb6adce0e2a4d7544c2 is 1080, key is row0022/info:/1733844667277/Put/seqid=0
2024-12-10T15:31:19,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741849_1025 (size=12509)
2024-12-10T15:31:19,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741849_1025 (size=12509)
2024-12-10T15:31:19,327 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/e618f54eecea4bb6adce0e2a4d7544c2
2024-12-10T15:31:19,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/e618f54eecea4bb6adce0e2a4d7544c2 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/e618f54eecea4bb6adce0e2a4d7544c2
2024-12-10T15:31:19,347 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/e618f54eecea4bb6adce0e2a4d7544c2, entries=7, sequenceid=42, filesize=12.2 K
2024-12-10T15:31:19,349 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 36ms, sequenceid=42, compaction requested=false
2024-12-10T15:31:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:

2024-12-10T15:31:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-10T15:31:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/1399045e5d4d4b4884f4e78b5601406b because midkey is the same as first or last row
2024-12-10T15:31:20,713 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:31:25,200 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 3eb3a6d6d7ebe91c66d296faa7fd4edc, had cached 0 bytes from a total of 40219
2024-12-10T15:31:27,333 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:31:27,334 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:31:27,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:27,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:27,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:27,341 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:31:27,341 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:31:27,341 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=368815740, stopped=false
2024-12-10T15:31:27,342 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,33141,1733844591575
2024-12-10T15:31:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:27,388 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:31:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:27,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:27,388 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:31:27,388 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:27,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:27,389 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,45265,1733844592430' *****
2024-12-10T15:31:27,389 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:27,389 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:31:27,389 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:27,389 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:31:27,389 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:31:27,390 INFO  [RS:0;d6bcd7308055:45265 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:31:27,390 INFO  [RS:0;d6bcd7308055:45265 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:31:27,390 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(3091): Received CLOSE for 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:31:27,391 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,45265,1733844592430
2024-12-10T15:31:27,391 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:31:27,391 INFO  [RS:0;d6bcd7308055:45265 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:45265.
2024-12-10T15:31:27,391 DEBUG [RS:0;d6bcd7308055:45265 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:27,391 DEBUG [RS:0;d6bcd7308055:45265 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:27,391 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3eb3a6d6d7ebe91c66d296faa7fd4edc, disabling compactions & flushes
2024-12-10T15:31:27,392 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:27,392 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:31:27,392 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:31:27,392 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:27,392 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:31:27,392 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc. after waiting 0 ms
2024-12-10T15:31:27,392 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:27,392 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:31:27,392 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-10T15:31:27,392 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T15:31:27,393 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:31:27,393 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1325): Online Regions={3eb3a6d6d7ebe91c66d296faa7fd4edc=TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc., 1588230740=hbase:meta,,1.1588230740}
2024-12-10T15:31:27,393 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:31:27,393 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:31:27,393 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:31:27,393 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:31:27,393 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-10T15:31:27,393 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3eb3a6d6d7ebe91c66d296faa7fd4edc
2024-12-10T15:31:27,408 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/fe552004440245f1abc877d3ad3578fe is 1080, key is row0029/info:/1733844681317/Put/seqid=0
2024-12-10T15:31:27,425 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/info/5a743aeb65da49fda2cc6dcfff870fba is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc./info:regioninfo/1733844595226/Put/seqid=0
2024-12-10T15:31:27,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741850_1026 (size=8193)
2024-12-10T15:31:27,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741850_1026 (size=8193)
2024-12-10T15:31:27,436 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/fe552004440245f1abc877d3ad3578fe
2024-12-10T15:31:27,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741851_1027 (size=7016)
2024-12-10T15:31:27,442 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/info/5a743aeb65da49fda2cc6dcfff870fba
2024-12-10T15:31:27,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741851_1027 (size=7016)
2024-12-10T15:31:27,449 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/.tmp/info/fe552004440245f1abc877d3ad3578fe as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/fe552004440245f1abc877d3ad3578fe
2024-12-10T15:31:27,462 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/fe552004440245f1abc877d3ad3578fe, entries=3, sequenceid=48, filesize=8.0 K
2024-12-10T15:31:27,464 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 72ms, sequenceid=48, compaction requested=true
2024-12-10T15:31:27,465 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2] to archive
2024-12-10T15:31:27,469 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:31:27,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/76539b9db7334d3893d123a254956603
2024-12-10T15:31:27,476 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/5d09338089ac4027a1e9c5004dd4de86
2024-12-10T15:31:27,477 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/ns/fed2e7f304c542ce945c27e66acb36c0 is 43, key is default/ns:d/1733844594435/Put/seqid=0
2024-12-10T15:31:27,477 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2 to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/info/57e2fd9fe13d4c6b86da54693253a1d2
2024-12-10T15:31:27,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741852_1028 (size=5153)
2024-12-10T15:31:27,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741852_1028 (size=5153)
2024-12-10T15:31:27,496 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/ns/fed2e7f304c542ce945c27e66acb36c0
2024-12-10T15:31:27,492 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d6bcd7308055:33141 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T15:31:27,499 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [76539b9db7334d3893d123a254956603=12509, 5d09338089ac4027a1e9c5004dd4de86=12509, 57e2fd9fe13d4c6b86da54693253a1d2=12509]
2024-12-10T15:31:27,508 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/default/TestLogRolling-testSlowSyncLogRolling/3eb3a6d6d7ebe91c66d296faa7fd4edc/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-10T15:31:27,512 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:27,512 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3eb3a6d6d7ebe91c66d296faa7fd4edc:
Waiting for close lock at 1733844687391Running coprocessor pre-close hooks at 1733844687391Disabling compacts and flushes for region at 1733844687391Disabling writes for close at 1733844687392 (+1 ms)Obtaining lock to block concurrent updates at 1733844687392Preparing flush snapshotting stores in 3eb3a6d6d7ebe91c66d296faa7fd4edc at 1733844687392Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733844687393 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc. at 1733844687394 (+1 ms)Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc/info: creating writer at 1733844687394Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc/info: appending metadata at 1733844687407 (+13 ms)Flushing 3eb3a6d6d7ebe91c66d296faa7fd4edc/info: closing flushed file at 1733844687407Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@366da0d8: reopening flushed file at 1733844687448 (+41 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 3eb3a6d6d7ebe91c66d296faa7fd4edc in 72ms, sequenceid=48, compaction requested=true at 1733844687464 (+16 ms)Writing region close event to WAL at 1733844687500 (+36 ms)Running coprocessor post-close hooks at 1733844687510 (+10 ms)Closed at 1733844687512 (+2 ms)
2024-12-10T15:31:27,513 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733844594749.3eb3a6d6d7ebe91c66d296faa7fd4edc.
2024-12-10T15:31:27,530 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/table/e0584afb32974a8491911c0e11cf67fd is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733844595241/Put/seqid=0
2024-12-10T15:31:27,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741853_1029 (size=5396)
2024-12-10T15:31:27,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741853_1029 (size=5396)
2024-12-10T15:31:27,556 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/table/e0584afb32974a8491911c0e11cf67fd
2024-12-10T15:31:27,568 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/info/5a743aeb65da49fda2cc6dcfff870fba as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/info/5a743aeb65da49fda2cc6dcfff870fba
2024-12-10T15:31:27,578 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/info/5a743aeb65da49fda2cc6dcfff870fba, entries=10, sequenceid=11, filesize=6.9 K
2024-12-10T15:31:27,580 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/ns/fed2e7f304c542ce945c27e66acb36c0 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/ns/fed2e7f304c542ce945c27e66acb36c0
2024-12-10T15:31:27,591 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/ns/fed2e7f304c542ce945c27e66acb36c0, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T15:31:27,593 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/.tmp/table/e0584afb32974a8491911c0e11cf67fd as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/table/e0584afb32974a8491911c0e11cf67fd
2024-12-10T15:31:27,593 DEBUG [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T15:31:27,606 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/table/e0584afb32974a8491911c0e11cf67fd, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T15:31:27,608 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 214ms, sequenceid=11, compaction requested=false
2024-12-10T15:31:27,619 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T15:31:27,620 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:31:27,621 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:27,621 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844687392Running coprocessor pre-close hooks at 1733844687392Disabling compacts and flushes for region at 1733844687393 (+1 ms)Disabling writes for close at 1733844687393Obtaining lock to block concurrent updates at 1733844687393Preparing flush snapshotting stores in 1588230740 at 1733844687393Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733844687394 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733844687395 (+1 ms)Flushing 1588230740/info: creating writer at 1733844687395Flushing 1588230740/info: appending metadata at 1733844687424 (+29 ms)Flushing 1588230740/info: closing flushed file at 1733844687424Flushing 1588230740/ns: creating writer at 1733844687455 (+31 ms)Flushing 1588230740/ns: appending metadata at 1733844687476 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733844687477 (+1 ms)Flushing 1588230740/table: creating writer at 1733844687509 (+32 ms)Flushing 1588230740/table: appending metadata at 1733844687530 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733844687530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4728e24f: reopening flushed file at 1733844687566 (+36 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2640edcb: reopening flushed file at 1733844687578 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@435692b5: reopening flushed file at 1733844687591 (+13 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 214ms, sequenceid=11, compaction requested=false at 1733844687608 (+17 ms)Writing region close event to WAL at 1733844687613 (+5 ms)Running coprocessor post-close hooks at 1733844687620 (+7 ms)Closed at 1733844687621 (+1 ms)
2024-12-10T15:31:27,621 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:27,633 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:31:27,720 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T15:31:27,720 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T15:31:27,794 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,45265,1733844592430; all regions closed.
2024-12-10T15:31:27,795 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,796 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,796 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,796 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,796 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741834_1010 (size=3066)
2024-12-10T15:31:27,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741834_1010 (size=3066)
2024-12-10T15:31:27,802 DEBUG [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs
2024-12-10T15:31:27,802 INFO  [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C45265%2C1733844592430.meta:.meta(num 1733844594253)
2024-12-10T15:31:27,803 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,803 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,803 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,803 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,803 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:27,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741847_1023 (size=12695)
2024-12-10T15:31:27,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741847_1023 (size=12695)
2024-12-10T15:31:27,810 DEBUG [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/oldWALs
2024-12-10T15:31:27,810 INFO  [RS:0;d6bcd7308055:45265 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C45265%2C1733844592430:(num 1733844667276)
2024-12-10T15:31:27,810 DEBUG [RS:0;d6bcd7308055:45265 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:27,810 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:31:27,810 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:31:27,810 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:31:27,810 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:31:27,810 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:31:27,811 INFO  [RS:0;d6bcd7308055:45265 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45265
2024-12-10T15:31:27,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,45265,1733844592430
2024-12-10T15:31:27,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:31:27,819 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:31:27,820 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,45265,1733844592430]
2024-12-10T15:31:27,924 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,45265,1733844592430 already deleted, retry=false
2024-12-10T15:31:27,925 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,45265,1733844592430 expired; onlineServers=0
2024-12-10T15:31:27,925 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,33141,1733844591575' *****
2024-12-10T15:31:27,925 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:31:27,925 INFO  [M:0;d6bcd7308055:33141 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:31:27,925 INFO  [M:0;d6bcd7308055:33141 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:31:27,925 DEBUG [M:0;d6bcd7308055:33141 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:31:27,926 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:31:27,926 DEBUG [M:0;d6bcd7308055:33141 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:31:27,926 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844593483 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844593483,5,FailOnTimeoutGroup]
2024-12-10T15:31:27,926 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844593482 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844593482,5,FailOnTimeoutGroup]
2024-12-10T15:31:27,926 INFO  [M:0;d6bcd7308055:33141 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:31:27,926 INFO  [M:0;d6bcd7308055:33141 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:31:27,926 DEBUG [M:0;d6bcd7308055:33141 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:31:27,926 INFO  [M:0;d6bcd7308055:33141 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:31:27,926 INFO  [M:0;d6bcd7308055:33141 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:31:27,927 INFO  [M:0;d6bcd7308055:33141 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:31:27,927 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:31:27,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:31:27,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:27,945 DEBUG [M:0;d6bcd7308055:33141 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-10T15:31:27,946 DEBUG [M:0;d6bcd7308055:33141 {}] master.ActiveMasterManager(353): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-10T15:31:27,946 INFO  [M:0;d6bcd7308055:33141 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/.lastflushedseqids
2024-12-10T15:31:27,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741854_1030 (size=130)
2024-12-10T15:31:27,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741854_1030 (size=130)
2024-12-10T15:31:27,963 INFO  [M:0;d6bcd7308055:33141 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:31:27,963 INFO  [M:0;d6bcd7308055:33141 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:31:27,964 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:31:27,964 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:27,964 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:27,964 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:31:27,964 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:27,964 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-12-10T15:31:27,990 DEBUG [M:0;d6bcd7308055:33141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f81934b90b6a481486f9ad727ac65cbc is 82, key is hbase:meta,,1/info:regioninfo/1733844594327/Put/seqid=0
2024-12-10T15:31:27,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741855_1031 (size=5672)
2024-12-10T15:31:27,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741855_1031 (size=5672)
2024-12-10T15:31:27,999 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f81934b90b6a481486f9ad727ac65cbc
2024-12-10T15:31:28,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:28,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45265-0x1001058e7790001, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:28,015 INFO  [RS:0;d6bcd7308055:45265 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:31:28,015 INFO  [RS:0;d6bcd7308055:45265 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,45265,1733844592430; zookeeper connection closed.
2024-12-10T15:31:28,016 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f326779 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f326779
2024-12-10T15:31:28,016 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:31:28,027 DEBUG [M:0;d6bcd7308055:33141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8f375ffee37b4ef98f12c744ed240b8c is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733844595248/Put/seqid=0
2024-12-10T15:31:28,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741856_1032 (size=6247)
2024-12-10T15:31:28,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741856_1032 (size=6247)
2024-12-10T15:31:28,035 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8f375ffee37b4ef98f12c744ed240b8c
2024-12-10T15:31:28,044 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8f375ffee37b4ef98f12c744ed240b8c
2024-12-10T15:31:28,067 DEBUG [M:0;d6bcd7308055:33141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab7b5f103e1d4c56aaf8df52eecd246f is 69, key is d6bcd7308055,45265,1733844592430/rs:state/1733844593554/Put/seqid=0
2024-12-10T15:31:28,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741857_1033 (size=5156)
2024-12-10T15:31:28,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741857_1033 (size=5156)
2024-12-10T15:31:28,082 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab7b5f103e1d4c56aaf8df52eecd246f
2024-12-10T15:31:28,115 DEBUG [M:0;d6bcd7308055:33141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c0a69d2a19834b488b6d740417ebd5f2 is 52, key is load_balancer_on/state:d/1733844594703/Put/seqid=0
2024-12-10T15:31:28,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741858_1034 (size=5056)
2024-12-10T15:31:28,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741858_1034 (size=5056)
2024-12-10T15:31:28,123 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c0a69d2a19834b488b6d740417ebd5f2
2024-12-10T15:31:28,133 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f81934b90b6a481486f9ad727ac65cbc as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f81934b90b6a481486f9ad727ac65cbc
2024-12-10T15:31:28,141 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f81934b90b6a481486f9ad727ac65cbc, entries=8, sequenceid=59, filesize=5.5 K
2024-12-10T15:31:28,142 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8f375ffee37b4ef98f12c744ed240b8c as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8f375ffee37b4ef98f12c744ed240b8c
2024-12-10T15:31:28,152 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8f375ffee37b4ef98f12c744ed240b8c
2024-12-10T15:31:28,152 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8f375ffee37b4ef98f12c744ed240b8c, entries=6, sequenceid=59, filesize=6.1 K
2024-12-10T15:31:28,156 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab7b5f103e1d4c56aaf8df52eecd246f as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab7b5f103e1d4c56aaf8df52eecd246f
2024-12-10T15:31:28,166 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab7b5f103e1d4c56aaf8df52eecd246f, entries=1, sequenceid=59, filesize=5.0 K
2024-12-10T15:31:28,168 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c0a69d2a19834b488b6d740417ebd5f2 as hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c0a69d2a19834b488b6d740417ebd5f2
2024-12-10T15:31:28,178 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c0a69d2a19834b488b6d740417ebd5f2, entries=1, sequenceid=59, filesize=4.9 K
2024-12-10T15:31:28,180 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 216ms, sequenceid=59, compaction requested=false
2024-12-10T15:31:28,184 INFO  [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:28,184 DEBUG [M:0;d6bcd7308055:33141 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844687964Disabling compacts and flushes for region at 1733844687964Disabling writes for close at 1733844687964Obtaining lock to block concurrent updates at 1733844687964Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844687964Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733844687965 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844687966 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844687966Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844687989 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844687989Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844688008 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844688026 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844688026Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844688045 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844688066 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844688066Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844688091 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844688114 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844688114Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18965529: reopening flushed file at 1733844688132 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e6f0d6f: reopening flushed file at 1733844688141 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3391fbc3: reopening flushed file at 1733844688153 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a9da9a6: reopening flushed file at 1733844688167 (+14 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 216ms, sequenceid=59, compaction requested=false at 1733844688180 (+13 ms)Writing region close event to WAL at 1733844688184 (+4 ms)Closed at 1733844688184
2024-12-10T15:31:28,185 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:28,186 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:28,186 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:28,186 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:28,186 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:28,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32771 is added to blk_1073741830_1006 (size=27973)
2024-12-10T15:31:28,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34063 is added to blk_1073741830_1006 (size=27973)
2024-12-10T15:31:28,191 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:31:28,191 INFO  [M:0;d6bcd7308055:33141 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:31:28,191 INFO  [M:0;d6bcd7308055:33141 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33141
2024-12-10T15:31:28,191 INFO  [M:0;d6bcd7308055:33141 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:31:28,298 INFO  [M:0;d6bcd7308055:33141 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:31:28,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:28,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33141-0x1001058e7790000, quorum=127.0.0.1:55710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:28,303 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@78ec6c63{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:28,305 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@633713c3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:28,305 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:28,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29058af{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:28,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@154eec55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:28,309 WARN  [BP-2125556533-172.17.0.2-1733844587642 heartbeating to localhost/127.0.0.1:44521 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:28,309 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:28,309 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:28,309 WARN  [BP-2125556533-172.17.0.2-1733844587642 heartbeating to localhost/127.0.0.1:44521 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2125556533-172.17.0.2-1733844587642 (Datanode Uuid 30f110e1-e152-4654-bb26-4478652fcebc) service to localhost/127.0.0.1:44521
2024-12-10T15:31:28,311 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data3/current/BP-2125556533-172.17.0.2-1733844587642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:28,311 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data4/current/BP-2125556533-172.17.0.2-1733844587642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:28,311 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:28,314 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b32dfc5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:28,314 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43ebd249{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:28,314 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:28,314 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e911877{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:28,314 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@460757e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:28,316 WARN  [BP-2125556533-172.17.0.2-1733844587642 heartbeating to localhost/127.0.0.1:44521 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:28,316 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:28,316 WARN  [BP-2125556533-172.17.0.2-1733844587642 heartbeating to localhost/127.0.0.1:44521 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2125556533-172.17.0.2-1733844587642 (Datanode Uuid 64cd6da1-1fc7-44b2-917a-b934dbae9e3f) service to localhost/127.0.0.1:44521
2024-12-10T15:31:28,316 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:28,317 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data1/current/BP-2125556533-172.17.0.2-1733844587642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:28,317 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/cluster_8f1fe453-773d-a7d5-5d51-e36b268498d0/data/data2/current/BP-2125556533-172.17.0.2-1733844587642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:28,317 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:28,326 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c1a236c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:31:28,327 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a625720{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:28,327 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:28,327 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ae7f863{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:28,327 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c5202f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:28,339 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:31:28,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:31:28,400 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12)
Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44521
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:44521 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/d6bcd7308055:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:44521
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44521
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/d6bcd7308055:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44521
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44521
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:44521 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/d6bcd7308055:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:44521 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@5e1d09c6
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 305), ProcessCount=11 (was 11), AvailableMemoryMB=4957 (was 5521)
2024-12-10T15:31:28,408 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=4957
2024-12-10T15:31:28,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.log.dir so I do NOT create it in target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/39ab975c-5242-c276-5019-88eda3bfc075/hadoop.tmp.dir so I do NOT create it in target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310, deleteOnExit=true
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/test.cache.data in system properties and HBase conf
2024-12-10T15:31:28,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:31:28,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:31:28,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:31:28,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:31:28,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:31:28,410 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:31:28,410 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:31:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:31:28,412 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:31:28,412 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:31:28,426 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:31:28,814 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:28,824 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:28,827 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:28,828 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:28,828 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:31:28,829 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:28,831 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@198be6ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:28,832 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d77f6e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:28,964 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cfb76b5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/java.io.tmpdir/jetty-localhost-40481-hadoop-hdfs-3_4_1-tests_jar-_-any-9419376102954765406/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:31:28,965 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d4e0dcd{HTTP/1.1, (http/1.1)}{localhost:40481}
2024-12-10T15:31:28,965 INFO  [Time-limited test {}] server.Server(415): Started @103556ms
2024-12-10T15:31:28,986 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:31:29,256 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:29,261 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:29,265 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:29,266 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:29,266 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:31:29,269 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@182e1f9b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:29,270 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f34a541{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:29,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c5e9a2a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/java.io.tmpdir/jetty-localhost-44333-hadoop-hdfs-3_4_1-tests_jar-_-any-11914740205507943257/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:29,385 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@752130d7{HTTP/1.1, (http/1.1)}{localhost:44333}
2024-12-10T15:31:29,385 INFO  [Time-limited test {}] server.Server(415): Started @103976ms
2024-12-10T15:31:29,387 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:29,438 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:29,442 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:29,443 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:29,443 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:29,443 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:31:29,444 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f72cd9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:29,444 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@197ae97a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:29,556 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@72edffcb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/java.io.tmpdir/jetty-localhost-38593-hadoop-hdfs-3_4_1-tests_jar-_-any-10734688674910238736/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:29,557 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ba90336{HTTP/1.1, (http/1.1)}{localhost:38593}
2024-12-10T15:31:29,557 INFO  [Time-limited test {}] server.Server(415): Started @104147ms
2024-12-10T15:31:29,558 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:30,455 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data2/current/BP-959643529-172.17.0.2-1733844688440/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:30,455 WARN  [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data1/current/BP-959643529-172.17.0.2-1733844688440/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:30,474 WARN  [Thread-415 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:30,476 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x793f48cf3cc7ab49 with lease ID 0xe8df9d4953000ee6: Processing first storage report for DS-d76714cf-d090-4bc5-bc05-201a7d06d193 from datanode DatanodeRegistration(127.0.0.1:34051, datanodeUuid=39ed70db-26cc-4c2b-82cf-edb73adbe8e7, infoPort=42055, infoSecurePort=0, ipcPort=34275, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440)
2024-12-10T15:31:30,476 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x793f48cf3cc7ab49 with lease ID 0xe8df9d4953000ee6: from storage DS-d76714cf-d090-4bc5-bc05-201a7d06d193 node DatanodeRegistration(127.0.0.1:34051, datanodeUuid=39ed70db-26cc-4c2b-82cf-edb73adbe8e7, infoPort=42055, infoSecurePort=0, ipcPort=34275, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:31:30,477 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x793f48cf3cc7ab49 with lease ID 0xe8df9d4953000ee6: Processing first storage report for DS-daa7df40-6e08-4578-a39e-b3107bae60de from datanode DatanodeRegistration(127.0.0.1:34051, datanodeUuid=39ed70db-26cc-4c2b-82cf-edb73adbe8e7, infoPort=42055, infoSecurePort=0, ipcPort=34275, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440)
2024-12-10T15:31:30,477 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x793f48cf3cc7ab49 with lease ID 0xe8df9d4953000ee6: from storage DS-daa7df40-6e08-4578-a39e-b3107bae60de node DatanodeRegistration(127.0.0.1:34051, datanodeUuid=39ed70db-26cc-4c2b-82cf-edb73adbe8e7, infoPort=42055, infoSecurePort=0, ipcPort=34275, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:30,594 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data4/current/BP-959643529-172.17.0.2-1733844688440/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:30,594 WARN  [Thread-462 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data3/current/BP-959643529-172.17.0.2-1733844688440/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:30,614 WARN  [Thread-438 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:30,617 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x114ffa344621cdce with lease ID 0xe8df9d4953000ee7: Processing first storage report for DS-45a804cc-1480-4ac8-9544-937d20791323 from datanode DatanodeRegistration(127.0.0.1:33589, datanodeUuid=3af27908-7d15-436f-b686-affdd9bb7677, infoPort=45855, infoSecurePort=0, ipcPort=38921, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440)
2024-12-10T15:31:30,617 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x114ffa344621cdce with lease ID 0xe8df9d4953000ee7: from storage DS-45a804cc-1480-4ac8-9544-937d20791323 node DatanodeRegistration(127.0.0.1:33589, datanodeUuid=3af27908-7d15-436f-b686-affdd9bb7677, infoPort=45855, infoSecurePort=0, ipcPort=38921, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:30,617 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x114ffa344621cdce with lease ID 0xe8df9d4953000ee7: Processing first storage report for DS-9296f380-2105-4768-a111-2a154607d795 from datanode DatanodeRegistration(127.0.0.1:33589, datanodeUuid=3af27908-7d15-436f-b686-affdd9bb7677, infoPort=45855, infoSecurePort=0, ipcPort=38921, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440)
2024-12-10T15:31:30,617 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x114ffa344621cdce with lease ID 0xe8df9d4953000ee7: from storage DS-9296f380-2105-4768-a111-2a154607d795 node DatanodeRegistration(127.0.0.1:33589, datanodeUuid=3af27908-7d15-436f-b686-affdd9bb7677, infoPort=45855, infoSecurePort=0, ipcPort=38921, storageInfo=lv=-57;cid=testClusterID;nsid=1692747662;c=1733844688440), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:30,705 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145
2024-12-10T15:31:30,708 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/zookeeper_0, clientPort=55169, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:31:30,710 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55169
2024-12-10T15:31:30,710 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:30,712 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:30,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:31:30,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:31:30,741 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9 with version=8
2024-12-10T15:31:30,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:31:30,743 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:31:30,743 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:31:30,744 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:31:30,745 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35941
2024-12-10T15:31:30,746 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35941 connecting to ZooKeeper ensemble=127.0.0.1:55169
2024-12-10T15:31:30,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:359410x0, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:31:30,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35941-0x100105a6dda0000 connected
2024-12-10T15:31:30,971 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:30,974 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:30,978 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:30,979 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9, hbase.cluster.distributed=false
2024-12-10T15:31:30,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:31:30,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35941
2024-12-10T15:31:30,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35941
2024-12-10T15:31:30,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35941
2024-12-10T15:31:30,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35941
2024-12-10T15:31:30,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35941
2024-12-10T15:31:31,003 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:31:31,003 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:31:31,004 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:31:31,005 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44281
2024-12-10T15:31:31,007 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44281 connecting to ZooKeeper ensemble=127.0.0.1:55169
2024-12-10T15:31:31,009 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:31,012 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:31,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442810x0, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:31:31,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44281-0x100105a6dda0001 connected
2024-12-10T15:31:31,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:31,032 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:31:31,037 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:31:31,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:31:31,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:31:31,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44281
2024-12-10T15:31:31,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44281
2024-12-10T15:31:31,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44281
2024-12-10T15:31:31,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44281
2024-12-10T15:31:31,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44281
2024-12-10T15:31:31,082 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:35941
2024-12-10T15:31:31,085 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:31,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:31,098 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:31:31,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,108 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:31:31,108 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,35941,1733844690743 from backup master directory
2024-12-10T15:31:31,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:31,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,118 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:31:31,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:31,118 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,123 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/hbase.id] with ID: e6956818-eded-4cfc-a8fe-4e91bdc5c725
2024-12-10T15:31:31,123 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/.tmp/hbase.id
2024-12-10T15:31:31,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:31:31,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:31:31,132 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/.tmp/hbase.id]:[hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/hbase.id]
2024-12-10T15:31:31,156 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:31,156 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:31:31,159 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-12-10T15:31:31,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:31:31,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:31:31,180 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:31:31,181 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:31:31,181 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:31,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:31:31,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:31:31,193 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store
2024-12-10T15:31:31,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:31:31,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:31:31,204 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:31,204 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:31,204 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844691204Disabling compacts and flushes for region at 1733844691204Disabling writes for close at 1733844691204Writing region close event to WAL at 1733844691204Closed at 1733844691204
2024-12-10T15:31:31,206 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/.initializing
2024-12-10T15:31:31,206 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/WALs/d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,210 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C35941%2C1733844690743, suffix=, logDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/WALs/d6bcd7308055,35941,1733844690743, archiveDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/oldWALs, maxLogs=10
2024-12-10T15:31:31,210 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C35941%2C1733844690743.1733844691210
2024-12-10T15:31:31,217 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/WALs/d6bcd7308055,35941,1733844690743/d6bcd7308055%2C35941%2C1733844690743.1733844691210
2024-12-10T15:31:31,221 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42055:42055),(127.0.0.1/127.0.0.1:45855:45855)]
2024-12-10T15:31:31,222 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:31:31,222 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:31,222 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,222 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,224 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:31,226 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:31:31,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:31,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:31,229 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:31:31,229 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,230 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:31,231 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:31,234 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:31:31,234 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,235 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:31,235 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:31,238 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:31:31,238 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,239 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:31,239 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,240 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,241 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,243 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,243 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,243 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:31:31,246 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:31,249 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:31:31,250 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736561, jitterRate=-0.06341464817523956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:31:31,251 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844691222Initializing all the Stores at 1733844691224 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844691224Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844691224Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844691224Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844691224Cleaning up temporary data from old regions at 1733844691243 (+19 ms)Region opened successfully at 1733844691251 (+8 ms)
2024-12-10T15:31:31,252 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:31:31,257 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7222511f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:31:31,259 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:31:31,259 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:31:31,259 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:31:31,259 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:31:31,260 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:31:31,261 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:31:31,261 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:31:31,264 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:31:31,265 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:31:31,275 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:31:31,276 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:31:31,277 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:31:31,286 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:31:31,286 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:31:31,288 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:31:31,296 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:31:31,298 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:31:31,307 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:31:31,311 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:31:31,323 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:31:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:31,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,334 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,35941,1733844690743, sessionid=0x100105a6dda0000, setting cluster-up flag (Was=false)
2024-12-10T15:31:31,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,386 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:31:31,388 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:31,439 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:31:31,440 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,35941,1733844690743
2024-12-10T15:31:31,442 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:31:31,445 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:31,445 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:31:31,445 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:31:31,446 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,35941,1733844690743
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:31:31,448 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,449 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:31:31,449 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844721449
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:31:31,450 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,451 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:31,451 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:31:31,451 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:31:31,451 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:31:31,451 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:31:31,451 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:31:31,452 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:31:31,452 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844691452,5,FailOnTimeoutGroup]
2024-12-10T15:31:31,452 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844691452,5,FailOnTimeoutGroup]
2024-12-10T15:31:31,452 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,452 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:31:31,453 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,453 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,453 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,453 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:31:31,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:31:31,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:31:31,466 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:31:31,467 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9
2024-12-10T15:31:31,471 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(746): ClusterId : e6956818-eded-4cfc-a8fe-4e91bdc5c725
2024-12-10T15:31:31,471 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:31:31,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:31:31,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:31,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:31:31,478 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:31:31,479 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:31:31,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,480 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:31,480 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:31:31,482 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:31:31,482 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:31:31,482 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:31:31,482 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:31,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:31:31,485 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:31:31,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:31,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:31:31,488 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:31:31,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:31,488 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:31,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:31:31,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740
2024-12-10T15:31:31,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740
2024-12-10T15:31:31,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:31:31,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:31:31,492 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:31:31,493 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:31:31,493 DEBUG [RS:0;d6bcd7308055:44281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d05b66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:31:31,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:31:31,497 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:31:31,498 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=828333, jitterRate=0.05328071117401123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:31:31,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844691476Initializing all the Stores at 1733844691477 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844691477Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844691477Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844691477Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844691477Cleaning up temporary data from old regions at 1733844691492 (+15 ms)Region opened successfully at 1733844691500 (+8 ms)
2024-12-10T15:31:31,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:31:31,500 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:31:31,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:31:31,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:31:31,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:31:31,508 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:44281
2024-12-10T15:31:31,508 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:31:31,508 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:31:31,508 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:31:31,509 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:31,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844691500Disabling compacts and flushes for region at 1733844691500Disabling writes for close at 1733844691500Writing region close event to WAL at 1733844691509 (+9 ms)Closed at 1733844691509
2024-12-10T15:31:31,509 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,35941,1733844690743 with port=44281, startcode=1733844691003
2024-12-10T15:31:31,509 DEBUG [RS:0;d6bcd7308055:44281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:31:31,510 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:31,510 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:31:31,511 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:31:31,512 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58497, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:31:31,513 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:31:31,513 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35941 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,513 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35941 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,515 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:31:31,515 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9
2024-12-10T15:31:31,515 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35539
2024-12-10T15:31:31,515 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:31:31,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:31:31,529 DEBUG [RS:0;d6bcd7308055:44281 {}] zookeeper.ZKUtil(111): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,529 WARN  [RS:0;d6bcd7308055:44281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:31:31,529 INFO  [RS:0;d6bcd7308055:44281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:31,529 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/WALs/d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,529 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,44281,1733844691003]
2024-12-10T15:31:31,534 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:31:31,537 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:31:31,538 INFO  [RS:0;d6bcd7308055:44281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:31:31,538 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,541 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:31:31,542 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:31:31,542 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,542 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,542 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:31,543 DEBUG [RS:0;d6bcd7308055:44281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,544 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44281,1733844691003-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:31:31,569 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:31:31,569 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44281,1733844691003-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,570 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,570 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.Replication(171): d6bcd7308055,44281,1733844691003 started
2024-12-10T15:31:31,586 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:31,587 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,44281,1733844691003, RpcServer on d6bcd7308055/172.17.0.2:44281, sessionid=0x100105a6dda0001
2024-12-10T15:31:31,587 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:31:31,587 DEBUG [RS:0;d6bcd7308055:44281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,587 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44281,1733844691003'
2024-12-10T15:31:31,587 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:31:31,588 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:31:31,588 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:31:31,588 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:31:31,589 DEBUG [RS:0;d6bcd7308055:44281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,589 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44281,1733844691003'
2024-12-10T15:31:31,589 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:31:31,589 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:31:31,590 DEBUG [RS:0;d6bcd7308055:44281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:31:31,590 INFO  [RS:0;d6bcd7308055:44281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:31:31,590 INFO  [RS:0;d6bcd7308055:44281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:31:31,665 WARN  [d6bcd7308055:35941 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:31:31,692 INFO  [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44281%2C1733844691003, suffix=, logDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/WALs/d6bcd7308055,44281,1733844691003, archiveDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/oldWALs, maxLogs=32
2024-12-10T15:31:31,694 INFO  [RS:0;d6bcd7308055:44281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44281%2C1733844691003.1733844691694
2024-12-10T15:31:31,702 INFO  [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/WALs/d6bcd7308055,44281,1733844691003/d6bcd7308055%2C44281%2C1733844691003.1733844691694
2024-12-10T15:31:31,704 DEBUG [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45855:45855),(127.0.0.1/127.0.0.1:42055:42055)]
2024-12-10T15:31:31,915 DEBUG [d6bcd7308055:35941 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:31:31,916 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,44281,1733844691003
2024-12-10T15:31:31,918 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44281,1733844691003, state=OPENING
2024-12-10T15:31:31,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:31:31,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:31:31,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-10T15:31:32,102 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:31:32,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:32,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:32,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:32,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:32,129 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:31:32,129 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44281,1733844691003}]
2024-12-10T15:31:32,284 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:31:32,287 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36439, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:31:32,292 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:31:32,292 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:32,295 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44281%2C1733844691003.meta, suffix=.meta, logDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/WALs/d6bcd7308055,44281,1733844691003, archiveDir=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/oldWALs, maxLogs=32
2024-12-10T15:31:32,298 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44281%2C1733844691003.meta.1733844692297.meta
2024-12-10T15:31:32,305 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/WALs/d6bcd7308055,44281,1733844691003/d6bcd7308055%2C44281%2C1733844691003.meta.1733844692297.meta
2024-12-10T15:31:32,306 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42055:42055),(127.0.0.1/127.0.0.1:45855:45855)]
2024-12-10T15:31:32,307 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:31:32,307 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:31:32,307 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:31:32,307 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:31:32,307 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:31:32,307 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:32,308 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:31:32,308 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:31:32,309 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:31:32,311 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:31:32,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:32,312 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:32,312 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:31:32,314 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:31:32,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:32,315 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:32,315 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:31:32,317 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:31:32,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:32,318 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:32,318 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:31:32,319 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:31:32,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:32,320 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:32,320 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:31:32,321 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740
2024-12-10T15:31:32,323 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740
2024-12-10T15:31:32,324 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:31:32,324 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:31:32,325 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:31:32,327 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:31:32,329 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738579, jitterRate=-0.06084839999675751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:31:32,329 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:31:32,330 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844692308Writing region info on filesystem at 1733844692308Initializing all the Stores at 1733844692309 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844692309Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844692309Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844692309Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844692309Cleaning up temporary data from old regions at 1733844692324 (+15 ms)Running coprocessor post-open hooks at 1733844692329 (+5 ms)Region opened successfully at 1733844692330 (+1 ms)
2024-12-10T15:31:32,331 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844692284
2024-12-10T15:31:32,334 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:31:32,335 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:31:32,336 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,44281,1733844691003
2024-12-10T15:31:32,337 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44281,1733844691003, state=OPEN
2024-12-10T15:31:32,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:31:32,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:31:32,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:32,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:32,383 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,44281,1733844691003
2024-12-10T15:31:32,387 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:31:32,388 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44281,1733844691003 in 254 msec
2024-12-10T15:31:32,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:31:32,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 877 msec
2024-12-10T15:31:32,396 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:32,396 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:31:32,398 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:31:32,398 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44281,1733844691003, seqNum=-1]
2024-12-10T15:31:32,399 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:31:32,400 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:31:32,411 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 965 msec
2024-12-10T15:31:32,412 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844692412, completionTime=-1
2024-12-10T15:31:32,412 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:31:32,412 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:31:32,415 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:31:32,415 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844752415
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844812416
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:35941, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,416 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,419 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:31:32,425 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,430 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.312sec
2024-12-10T15:31:32,430 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:31:32,431 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:31:32,436 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:31:32,436 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:31:32,436 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35941,1733844690743-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:32,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b14ffe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:32,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,35941,-1 for getting cluster id
2024-12-10T15:31:32,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:31:32,474 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e6956818-eded-4cfc-a8fe-4e91bdc5c725'
2024-12-10T15:31:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:31:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e6956818-eded-4cfc-a8fe-4e91bdc5c725"

2024-12-10T15:31:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e0c936b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,35941,-1]
2024-12-10T15:31:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:31:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,491 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:31:32,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109c38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:32,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:31:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44281,1733844691003, seqNum=-1]
2024-12-10T15:31:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:31:32,504 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:31:32,507 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,35941,1733844690743
2024-12-10T15:31:32,508 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:32,511 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:31:32,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:31:32,511 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:31:32,512 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:32,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,512 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:31:32,513 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:31:32,513 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=355378974, stopped=false
2024-12-10T15:31:32,513 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,35941,1733844690743
2024-12-10T15:31:32,514 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:32,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:32,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:32,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:32,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:32,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:32,585 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:31:32,585 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:31:32,585 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:32,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,585 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,44281,1733844691003' *****
2024-12-10T15:31:32,586 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:31:32,587 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,44281,1733844691003
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:31:32,587 INFO  [RS:0;d6bcd7308055:44281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:44281.
2024-12-10T15:31:32,588 DEBUG [RS:0;d6bcd7308055:44281 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:31:32,588 DEBUG [RS:0;d6bcd7308055:44281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,588 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:31:32,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:32,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:32,592 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:31:32,592 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:31:32,592 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:31:32,597 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T15:31:32,597 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-10T15:31:32,597 DEBUG [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T15:31:32,597 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:31:32,598 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:31:32,598 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:31:32,599 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:31:32,599 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:31:32,600 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-10T15:31:32,632 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T15:31:32,632 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T15:31:32,633 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/.tmp/ns/b24b9da4ed0f49f58db0e77b12efbd93 is 43, key is default/ns:d/1733844692401/Put/seqid=0
2024-12-10T15:31:32,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741835_1011 (size=5153)
2024-12-10T15:31:32,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741835_1011 (size=5153)
2024-12-10T15:31:32,644 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/.tmp/ns/b24b9da4ed0f49f58db0e77b12efbd93
2024-12-10T15:31:32,654 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/.tmp/ns/b24b9da4ed0f49f58db0e77b12efbd93 as hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/ns/b24b9da4ed0f49f58db0e77b12efbd93
2024-12-10T15:31:32,663 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/ns/b24b9da4ed0f49f58db0e77b12efbd93, entries=2, sequenceid=6, filesize=5.0 K
2024-12-10T15:31:32,664 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=6, compaction requested=false
2024-12-10T15:31:32,664 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T15:31:32,671 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-10T15:31:32,673 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:31:32,673 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:32,673 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844692597Running coprocessor pre-close hooks at 1733844692597Disabling compacts and flushes for region at 1733844692597Disabling writes for close at 1733844692599 (+2 ms)Obtaining lock to block concurrent updates at 1733844692600 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733844692600Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733844692601 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733844692602 (+1 ms)Flushing 1588230740/ns: creating writer at 1733844692602Flushing 1588230740/ns: appending metadata at 1733844692632 (+30 ms)Flushing 1588230740/ns: closing flushed file at 1733844692632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b1a0cb2: reopening flushed file at 1733844692652 (+20 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=6, compaction requested=false at 1733844692664 (+12 ms)Writing region close event to WAL at 1733844692666 (+2 ms)Running coprocessor post-close hooks at 1733844692672 (+6 ms)Closed at 1733844692673 (+1 ms)
2024-12-10T15:31:32,673 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:32,797 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,44281,1733844691003; all regions closed.
2024-12-10T15:31:32,798 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,798 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,799 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,799 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,799 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741834_1010 (size=1152)
2024-12-10T15:31:32,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741834_1010 (size=1152)
2024-12-10T15:31:32,807 DEBUG [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/oldWALs
2024-12-10T15:31:32,807 INFO  [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44281%2C1733844691003.meta:.meta(num 1733844692297)
2024-12-10T15:31:32,808 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,808 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,808 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,808 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,808 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:32,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741833_1009 (size=93)
2024-12-10T15:31:32,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741833_1009 (size=93)
2024-12-10T15:31:32,813 DEBUG [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/oldWALs
2024-12-10T15:31:32,813 INFO  [RS:0;d6bcd7308055:44281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44281%2C1733844691003:(num 1733844691694)
2024-12-10T15:31:32,813 DEBUG [RS:0;d6bcd7308055:44281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:32,813 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:31:32,813 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:31:32,814 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:31:32,814 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:31:32,814 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:31:32,814 INFO  [RS:0;d6bcd7308055:44281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44281
2024-12-10T15:31:32,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,44281,1733844691003
2024-12-10T15:31:32,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:31:32,844 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:31:32,854 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,44281,1733844691003]
2024-12-10T15:31:32,865 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,44281,1733844691003 already deleted, retry=false
2024-12-10T15:31:32,865 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,44281,1733844691003 expired; onlineServers=0
2024-12-10T15:31:32,865 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,35941,1733844690743' *****
2024-12-10T15:31:32,865 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:31:32,865 INFO  [M:0;d6bcd7308055:35941 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:31:32,865 INFO  [M:0;d6bcd7308055:35941 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:31:32,865 DEBUG [M:0;d6bcd7308055:35941 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:31:32,866 DEBUG [M:0;d6bcd7308055:35941 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:31:32,866 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:31:32,866 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844691452 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844691452,5,FailOnTimeoutGroup]
2024-12-10T15:31:32,866 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844691452 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844691452,5,FailOnTimeoutGroup]
2024-12-10T15:31:32,866 INFO  [M:0;d6bcd7308055:35941 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:31:32,866 INFO  [M:0;d6bcd7308055:35941 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:31:32,866 DEBUG [M:0;d6bcd7308055:35941 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:31:32,866 INFO  [M:0;d6bcd7308055:35941 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:31:32,866 INFO  [M:0;d6bcd7308055:35941 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:31:32,867 INFO  [M:0;d6bcd7308055:35941 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:31:32,867 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:31:32,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:31:32,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:32,876 DEBUG [M:0;d6bcd7308055:35941 {}] zookeeper.ZKUtil(347): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T15:31:32,876 WARN  [M:0;d6bcd7308055:35941 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T15:31:32,877 INFO  [M:0;d6bcd7308055:35941 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/.lastflushedseqids
2024-12-10T15:31:32,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741836_1012 (size=99)
2024-12-10T15:31:32,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741836_1012 (size=99)
2024-12-10T15:31:32,886 INFO  [M:0;d6bcd7308055:35941 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:31:32,886 INFO  [M:0;d6bcd7308055:35941 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:31:32,886 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:31:32,886 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:32,886 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:32,886 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:31:32,886 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:32,886 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-10T15:31:32,906 DEBUG [M:0;d6bcd7308055:35941 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19173c77ec489b92e64688b298a3dc is 82, key is hbase:meta,,1/info:regioninfo/1733844692336/Put/seqid=0
2024-12-10T15:31:32,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741837_1013 (size=5672)
2024-12-10T15:31:32,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741837_1013 (size=5672)
2024-12-10T15:31:32,914 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19173c77ec489b92e64688b298a3dc
2024-12-10T15:31:32,935 DEBUG [M:0;d6bcd7308055:35941 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/012d23de39f14e0f95d60acb9c854bc7 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733844692410/Put/seqid=0
2024-12-10T15:31:32,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741838_1014 (size=5275)
2024-12-10T15:31:32,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741838_1014 (size=5275)
2024-12-10T15:31:32,941 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/012d23de39f14e0f95d60acb9c854bc7
2024-12-10T15:31:32,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:32,955 INFO  [RS:0;d6bcd7308055:44281 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:31:32,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44281-0x100105a6dda0001, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:32,955 INFO  [RS:0;d6bcd7308055:44281 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,44281,1733844691003; zookeeper connection closed.
2024-12-10T15:31:32,955 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6aba2761 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6aba2761
2024-12-10T15:31:32,955 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:31:32,965 DEBUG [M:0;d6bcd7308055:35941 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/072071775c2a4529bede110336a1eb27 is 69, key is d6bcd7308055,44281,1733844691003/rs:state/1733844691514/Put/seqid=0
2024-12-10T15:31:32,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741839_1015 (size=5156)
2024-12-10T15:31:32,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741839_1015 (size=5156)
2024-12-10T15:31:32,971 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/072071775c2a4529bede110336a1eb27
2024-12-10T15:31:32,994 DEBUG [M:0;d6bcd7308055:35941 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0c924e6769c74b86884eaad5f041b182 is 52, key is load_balancer_on/state:d/1733844692509/Put/seqid=0
2024-12-10T15:31:32,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741840_1016 (size=5056)
2024-12-10T15:31:32,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741840_1016 (size=5056)
2024-12-10T15:31:33,000 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0c924e6769c74b86884eaad5f041b182
2024-12-10T15:31:33,008 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd19173c77ec489b92e64688b298a3dc as hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd19173c77ec489b92e64688b298a3dc
2024-12-10T15:31:33,014 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd19173c77ec489b92e64688b298a3dc, entries=8, sequenceid=29, filesize=5.5 K
2024-12-10T15:31:33,016 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/012d23de39f14e0f95d60acb9c854bc7 as hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/012d23de39f14e0f95d60acb9c854bc7
2024-12-10T15:31:33,022 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/012d23de39f14e0f95d60acb9c854bc7, entries=3, sequenceid=29, filesize=5.2 K
2024-12-10T15:31:33,024 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/072071775c2a4529bede110336a1eb27 as hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/072071775c2a4529bede110336a1eb27
2024-12-10T15:31:33,031 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/072071775c2a4529bede110336a1eb27, entries=1, sequenceid=29, filesize=5.0 K
2024-12-10T15:31:33,033 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0c924e6769c74b86884eaad5f041b182 as hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0c924e6769c74b86884eaad5f041b182
2024-12-10T15:31:33,037 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:31:33,040 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:33,041 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35539/user/jenkins/test-data/e72cfea2-9316-5814-767d-bed8ee9c62e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0c924e6769c74b86884eaad5f041b182, entries=1, sequenceid=29, filesize=4.9 K
2024-12-10T15:31:33,042 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=29, compaction requested=false
2024-12-10T15:31:33,044 INFO  [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:33,044 DEBUG [M:0;d6bcd7308055:35941 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844692886Disabling compacts and flushes for region at 1733844692886Disabling writes for close at 1733844692886Obtaining lock to block concurrent updates at 1733844692886Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844692886Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733844692887 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844692888 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844692888Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844692906 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844692906Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844692920 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844692935 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844692935Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844692946 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844692964 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844692964Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844692977 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844692993 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844692993Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15c9b26: reopening flushed file at 1733844693006 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27fa891b: reopening flushed file at 1733844693015 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a319ec: reopening flushed file at 1733844693023 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@485df0c2: reopening flushed file at 1733844693031 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=29, compaction requested=false at 1733844693042 (+11 ms)Writing region close event to WAL at 1733844693043 (+1 ms)Closed at 1733844693043
2024-12-10T15:31:33,044 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:33,044 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:33,044 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:33,044 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:33,045 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:33,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33589 is added to blk_1073741830_1006 (size=10311)
2024-12-10T15:31:33,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34051 is added to blk_1073741830_1006 (size=10311)
2024-12-10T15:31:33,048 INFO  [M:0;d6bcd7308055:35941 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:31:33,048 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:31:33,048 INFO  [M:0;d6bcd7308055:35941 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35941
2024-12-10T15:31:33,048 INFO  [M:0;d6bcd7308055:35941 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:31:33,062 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:33,160 INFO  [M:0;d6bcd7308055:35941 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:31:33,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:33,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35941-0x100105a6dda0000, quorum=127.0.0.1:55169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:31:33,162 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@72edffcb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:33,163 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ba90336{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:33,163 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:33,163 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@197ae97a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:33,163 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f72cd9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:33,164 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:33,164 WARN  [BP-959643529-172.17.0.2-1733844688440 heartbeating to localhost/127.0.0.1:35539 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:33,164 WARN  [BP-959643529-172.17.0.2-1733844688440 heartbeating to localhost/127.0.0.1:35539 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959643529-172.17.0.2-1733844688440 (Datanode Uuid 3af27908-7d15-436f-b686-affdd9bb7677) service to localhost/127.0.0.1:35539
2024-12-10T15:31:33,164 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:33,165 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data3/current/BP-959643529-172.17.0.2-1733844688440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:33,165 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data4/current/BP-959643529-172.17.0.2-1733844688440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:33,165 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:33,168 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c5e9a2a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:33,168 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@752130d7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:33,168 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:33,168 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f34a541{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:33,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@182e1f9b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:33,170 WARN  [BP-959643529-172.17.0.2-1733844688440 heartbeating to localhost/127.0.0.1:35539 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:33,170 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:33,170 WARN  [BP-959643529-172.17.0.2-1733844688440 heartbeating to localhost/127.0.0.1:35539 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959643529-172.17.0.2-1733844688440 (Datanode Uuid 39ed70db-26cc-4c2b-82cf-edb73adbe8e7) service to localhost/127.0.0.1:35539
2024-12-10T15:31:33,170 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:33,171 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data1/current/BP-959643529-172.17.0.2-1733844688440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:33,171 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/cluster_b5fbf6c4-ea57-8e2b-28e9-f3b54097c310/data/data2/current/BP-959643529-172.17.0.2-1733844688440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:33,171 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:33,177 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cfb76b5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:31:33,177 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d4e0dcd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:33,177 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:33,177 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d77f6e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:33,178 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@198be6ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:33,184 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.log.dir so I do NOT create it in target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c92964-93f5-6c03-5dcf-6d7eb9821145/hadoop.tmp.dir so I do NOT create it in target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab, deleteOnExit=true
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/test.cache.data in system properties and HBase conf
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:31:33,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:31:33,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:31:33,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:31:33,204 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:31:33,217 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:31:33,525 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:33,531 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:33,532 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:33,532 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:33,532 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:31:33,533 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:33,533 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f149dad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:33,533 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56d3681c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:33,553 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:31:33,636 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@148f816d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-38297-hadoop-hdfs-3_4_1-tests_jar-_-any-11443685732112856440/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:31:33,649 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59a85f94{HTTP/1.1, (http/1.1)}{localhost:38297}
2024-12-10T15:31:33,649 INFO  [Time-limited test {}] server.Server(415): Started @108240ms
2024-12-10T15:31:33,663 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:31:33,907 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:33,911 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:33,913 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:33,913 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:33,913 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:31:33,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6586e5e2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:33,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24f9668a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:34,021 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6256f4f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-43421-hadoop-hdfs-3_4_1-tests_jar-_-any-10322297660696253658/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:34,022 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@792ad53f{HTTP/1.1, (http/1.1)}{localhost:43421}
2024-12-10T15:31:34,022 INFO  [Time-limited test {}] server.Server(415): Started @108613ms
2024-12-10T15:31:34,024 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:34,085 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:34,089 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:34,093 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:34,093 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:34,093 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:31:34,094 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76a89480{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:34,094 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3351669c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:34,203 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75e3c7fe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-33489-hadoop-hdfs-3_4_1-tests_jar-_-any-16965758411658750426/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:34,204 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a8aaf72{HTTP/1.1, (http/1.1)}{localhost:33489}
2024-12-10T15:31:34,204 INFO  [Time-limited test {}] server.Server(415): Started @108795ms
2024-12-10T15:31:34,205 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:35,037 WARN  [Thread-670 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data1/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:35,037 WARN  [Thread-671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data2/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:35,061 WARN  [Thread-635 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:35,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7b15728ddc45f292 with lease ID 0xa3daa31765999bd9: Processing first storage report for DS-64f45c27-6145-4326-85b1-d289e2439498 from datanode DatanodeRegistration(127.0.0.1:36565, datanodeUuid=24d0e157-4045-4a75-a330-616f0c3e7300, infoPort=44155, infoSecurePort=0, ipcPort=35079, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:35,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7b15728ddc45f292 with lease ID 0xa3daa31765999bd9: from storage DS-64f45c27-6145-4326-85b1-d289e2439498 node DatanodeRegistration(127.0.0.1:36565, datanodeUuid=24d0e157-4045-4a75-a330-616f0c3e7300, infoPort=44155, infoSecurePort=0, ipcPort=35079, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:35,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7b15728ddc45f292 with lease ID 0xa3daa31765999bd9: Processing first storage report for DS-ff80de96-ac87-4aab-aadc-3e48d0651251 from datanode DatanodeRegistration(127.0.0.1:36565, datanodeUuid=24d0e157-4045-4a75-a330-616f0c3e7300, infoPort=44155, infoSecurePort=0, ipcPort=35079, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:35,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7b15728ddc45f292 with lease ID 0xa3daa31765999bd9: from storage DS-ff80de96-ac87-4aab-aadc-3e48d0651251 node DatanodeRegistration(127.0.0.1:36565, datanodeUuid=24d0e157-4045-4a75-a330-616f0c3e7300, infoPort=44155, infoSecurePort=0, ipcPort=35079, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:35,279 WARN  [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data3/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:35,293 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data4/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:35,309 WARN  [Thread-658 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:35,311 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdf57fad3441e2de7 with lease ID 0xa3daa31765999bda: Processing first storage report for DS-8bbd286c-9234-4134-a894-534f3213518a from datanode DatanodeRegistration(127.0.0.1:33249, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=46599, infoSecurePort=0, ipcPort=32989, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:35,311 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf57fad3441e2de7 with lease ID 0xa3daa31765999bda: from storage DS-8bbd286c-9234-4134-a894-534f3213518a node DatanodeRegistration(127.0.0.1:33249, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=46599, infoSecurePort=0, ipcPort=32989, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:35,311 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdf57fad3441e2de7 with lease ID 0xa3daa31765999bda: Processing first storage report for DS-b0b9f480-fb1c-4ee7-9c13-c9827e768128 from datanode DatanodeRegistration(127.0.0.1:33249, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=46599, infoSecurePort=0, ipcPort=32989, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:35,311 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf57fad3441e2de7 with lease ID 0xa3daa31765999bda: from storage DS-b0b9f480-fb1c-4ee7-9c13-c9827e768128 node DatanodeRegistration(127.0.0.1:33249, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=46599, infoSecurePort=0, ipcPort=32989, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:35,359 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69
2024-12-10T15:31:35,365 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/zookeeper_0, clientPort=56326, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:31:35,366 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56326
2024-12-10T15:31:35,367 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,369 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:31:35,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:31:35,382 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165 with version=8
2024-12-10T15:31:35,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:31:35,385 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:31:35,386 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:31:35,388 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44557
2024-12-10T15:31:35,389 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44557 connecting to ZooKeeper ensemble=127.0.0.1:56326
2024-12-10T15:31:35,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445570x0, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:31:35,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44557-0x100105a80090000 connected
2024-12-10T15:31:35,528 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,530 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:35,533 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165, hbase.cluster.distributed=false
2024-12-10T15:31:35,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:31:35,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44557
2024-12-10T15:31:35,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44557
2024-12-10T15:31:35,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44557
2024-12-10T15:31:35,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44557
2024-12-10T15:31:35,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44557
2024-12-10T15:31:35,553 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:31:35,554 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:31:35,555 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42517
2024-12-10T15:31:35,557 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42517 connecting to ZooKeeper ensemble=127.0.0.1:56326
2024-12-10T15:31:35,557 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,560 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425170x0, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:31:35,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425170x0, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:31:35,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42517-0x100105a80090001 connected
2024-12-10T15:31:35,576 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:31:35,577 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:31:35,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:31:35,578 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:31:35,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42517
2024-12-10T15:31:35,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42517
2024-12-10T15:31:35,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42517
2024-12-10T15:31:35,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42517
2024-12-10T15:31:35,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42517
2024-12-10T15:31:35,592 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:44557
2024-12-10T15:31:35,593 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:35,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:35,602 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:31:35,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,613 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:31:35,613 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,44557,1733844695385 from backup master directory
2024-12-10T15:31:35,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:35,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:31:35,623 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:31:35,623 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,630 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/hbase.id] with ID: cb3c78bb-d10d-461e-9a2e-2952e55b093c
2024-12-10T15:31:35,630 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/.tmp/hbase.id
2024-12-10T15:31:35,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:31:35,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:31:35,637 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/.tmp/hbase.id]:[hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/hbase.id]
2024-12-10T15:31:35,651 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:35,651 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:31:35,653 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-10T15:31:35,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:31:35,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:31:35,674 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:31:35,675 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:31:35,676 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:35,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:31:35,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:31:35,685 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store
2024-12-10T15:31:35,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:31:35,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:31:35,693 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:35,693 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:31:35,693 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844695693Disabling compacts and flushes for region at 1733844695693Disabling writes for close at 1733844695693Writing region close event to WAL at 1733844695693Closed at 1733844695693
2024-12-10T15:31:35,694 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/.initializing
2024-12-10T15:31:35,694 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,698 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44557%2C1733844695385, suffix=, logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385, archiveDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/oldWALs, maxLogs=10
2024-12-10T15:31:35,699 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44557%2C1733844695385.1733844695698
2024-12-10T15:31:35,705 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
2024-12-10T15:31:35,709 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46599:46599),(127.0.0.1/127.0.0.1:44155:44155)]
2024-12-10T15:31:35,710 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:31:35,710 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:35,710 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,710 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,712 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:35,713 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:31:35,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,714 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:35,714 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:35,715 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:31:35,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,716 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:35,716 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:35,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:31:35,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:35,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:31:35,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:31:35,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:35,720 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,721 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,722 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,723 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,724 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,724 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:31:35,726 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:31:35,728 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:31:35,729 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=758001, jitterRate=-0.036152422428131104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:31:35,730 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844695710Initializing all the Stores at 1733844695711 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844695711Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844695712 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844695712Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844695712Cleaning up temporary data from old regions at 1733844695724 (+12 ms)Region opened successfully at 1733844695730 (+6 ms)
2024-12-10T15:31:35,730 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:31:35,734 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca686f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:31:35,735 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:31:35,735 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:31:35,735 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:31:35,736 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:31:35,736 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:31:35,737 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:31:35,737 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:31:35,739 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:31:35,740 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:31:35,749 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:31:35,749 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:31:35,750 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:31:35,759 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:31:35,760 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:31:35,761 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:31:35,770 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:31:35,771 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:31:35,780 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:31:35,783 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:31:35,791 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:31:35,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:35,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:31:35,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,802 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,44557,1733844695385, sessionid=0x100105a80090000, setting cluster-up flag (Was=false)
2024-12-10T15:31:35,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,854 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:31:35,855 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:35,907 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:31:35,908 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,44557,1733844695385
2024-12-10T15:31:35,910 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:31:35,911 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:35,912 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:31:35,912 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:31:35,912 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,44557,1733844695385
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:31:35,913 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:35,913 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:31:35,914 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844725915
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:31:35,915 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:35,916 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:31:35,916 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:35,916 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:31:35,916 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:31:35,916 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:31:35,916 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:31:35,916 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:31:35,917 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,917 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:31:35,921 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844695916,5,FailOnTimeoutGroup]
2024-12-10T15:31:35,929 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844695921,5,FailOnTimeoutGroup]
2024-12-10T15:31:35,929 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:35,929 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:31:35,929 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:35,929 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:35,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:31:35,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:31:35,939 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:31:35,940 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165
2024-12-10T15:31:35,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:31:35,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:31:35,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:35,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:31:35,955 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:31:35,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,956 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:35,956 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:31:35,958 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:31:35,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,959 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:35,959 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:31:35,961 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:31:35,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:35,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:31:35,964 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:31:35,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:35,965 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:35,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:31:35,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740
2024-12-10T15:31:35,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740
2024-12-10T15:31:35,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:31:35,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:31:35,969 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:31:35,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:31:35,973 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:31:35,974 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784147, jitterRate=-0.002906486392021179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:31:35,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844695952Initializing all the Stores at 1733844695953 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844695953Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844695953Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844695953Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844695953Cleaning up temporary data from old regions at 1733844695968 (+15 ms)Region opened successfully at 1733844695974 (+6 ms)
2024-12-10T15:31:35,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:31:35,975 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:31:35,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:31:35,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:31:35,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:31:35,975 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:31:35,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844695975Disabling compacts and flushes for region at 1733844695975Disabling writes for close at 1733844695975Writing region close event to WAL at 1733844695975Closed at 1733844695975
2024-12-10T15:31:35,977 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:35,977 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:31:35,977 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:31:35,979 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:31:35,980 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:31:35,982 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(746): ClusterId : cb3c78bb-d10d-461e-9a2e-2952e55b093c
2024-12-10T15:31:35,982 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:31:35,992 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:31:35,992 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:31:36,003 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:31:36,003 DEBUG [RS:0;d6bcd7308055:42517 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e7e444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:31:36,015 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:42517
2024-12-10T15:31:36,015 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:31:36,016 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:31:36,016 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:31:36,016 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,44557,1733844695385 with port=42517, startcode=1733844695553
2024-12-10T15:31:36,017 DEBUG [RS:0;d6bcd7308055:42517 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:31:36,019 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42905, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:31:36,019 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44557 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,019 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44557 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,021 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165
2024-12-10T15:31:36,021 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34973
2024-12-10T15:31:36,021 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:31:36,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:31:36,103 DEBUG [RS:0;d6bcd7308055:42517 {}] zookeeper.ZKUtil(111): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,103 WARN  [RS:0;d6bcd7308055:42517 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:31:36,103 INFO  [RS:0;d6bcd7308055:42517 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:36,103 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,103 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,42517,1733844695553]
2024-12-10T15:31:36,111 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:31:36,117 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:31:36,125 INFO  [RS:0;d6bcd7308055:42517 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:31:36,125 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,125 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:31:36,126 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:31:36,127 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,127 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:36,128 DEBUG [RS:0;d6bcd7308055:42517 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,128 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,42517,1733844695553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:31:36,130 WARN  [d6bcd7308055:44557 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:31:36,150 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:31:36,150 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,42517,1733844695553-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,151 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,151 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.Replication(171): d6bcd7308055,42517,1733844695553 started
2024-12-10T15:31:36,187 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,187 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,42517,1733844695553, RpcServer on d6bcd7308055/172.17.0.2:42517, sessionid=0x100105a80090001
2024-12-10T15:31:36,187 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:31:36,187 DEBUG [RS:0;d6bcd7308055:42517 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,187 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,42517,1733844695553'
2024-12-10T15:31:36,187 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,42517,1733844695553'
2024-12-10T15:31:36,188 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:31:36,189 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:31:36,189 DEBUG [RS:0;d6bcd7308055:42517 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:31:36,189 INFO  [RS:0;d6bcd7308055:42517 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:31:36,189 INFO  [RS:0;d6bcd7308055:42517 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:31:36,293 INFO  [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C42517%2C1733844695553, suffix=, logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553, archiveDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs, maxLogs=32
2024-12-10T15:31:36,295 INFO  [RS:0;d6bcd7308055:42517 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C42517%2C1733844695553.1733844696295
2024-12-10T15:31:36,303 INFO  [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
2024-12-10T15:31:36,305 DEBUG [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44155:44155),(127.0.0.1/127.0.0.1:46599:46599)]
2024-12-10T15:31:36,380 DEBUG [d6bcd7308055:44557 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:31:36,381 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,383 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,42517,1733844695553, state=OPENING
2024-12-10T15:31:36,401 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:31:36,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:36,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:31:36,493 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:31:36,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:36,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:36,493 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,42517,1733844695553}]
2024-12-10T15:31:36,647 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:31:36,650 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60489, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:31:36,655 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:31:36,655 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:36,658 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C42517%2C1733844695553.meta, suffix=.meta, logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553, archiveDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs, maxLogs=32
2024-12-10T15:31:36,659 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
2024-12-10T15:31:36,664 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
2024-12-10T15:31:36,669 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46599:46599),(127.0.0.1/127.0.0.1:44155:44155)]
2024-12-10T15:31:36,670 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:31:36,670 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:31:36,670 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:31:36,670 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:31:36,670 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:31:36,670 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:36,671 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:31:36,671 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:31:36,672 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:31:36,673 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:31:36,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:36,674 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:36,674 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:31:36,675 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:31:36,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:36,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:36,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:31:36,677 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:31:36,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:36,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:36,678 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:31:36,678 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:31:36,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:36,679 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:31:36,679 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:31:36,680 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740
2024-12-10T15:31:36,681 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740
2024-12-10T15:31:36,683 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:31:36,683 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:31:36,683 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:31:36,685 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:31:36,686 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708249, jitterRate=-0.09941600263118744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:31:36,686 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:31:36,687 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844696671Writing region info on filesystem at 1733844696671Initializing all the Stores at 1733844696672 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844696672Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844696672Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844696672Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844696672Cleaning up temporary data from old regions at 1733844696683 (+11 ms)Running coprocessor post-open hooks at 1733844696686 (+3 ms)Region opened successfully at 1733844696686
2024-12-10T15:31:36,688 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844696647
2024-12-10T15:31:36,690 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:31:36,690 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:31:36,691 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,692 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,42517,1733844695553, state=OPEN
2024-12-10T15:31:36,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:31:36,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:31:36,731 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,42517,1733844695553
2024-12-10T15:31:36,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:36,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:31:36,735 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:31:36,736 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,42517,1733844695553 in 238 msec
2024-12-10T15:31:36,739 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:31:36,739 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 758 msec
2024-12-10T15:31:36,740 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:31:36,740 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:31:36,742 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:31:36,742 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,42517,1733844695553, seqNum=-1]
2024-12-10T15:31:36,742 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:31:36,743 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42191, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:31:36,750 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 837 msec
2024-12-10T15:31:36,750 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844696750, completionTime=-1
2024-12-10T15:31:36,750 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:31:36,750 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:31:36,752 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:31:36,752 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844756752
2024-12-10T15:31:36,752 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844816752
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:44557, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,753 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,754 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,755 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:31:36,757 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.134sec
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:31:36,758 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:31:36,760 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:31:36,760 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:31:36,760 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44557,1733844695385-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cea6c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:36,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,44557,-1 for getting cluster id
2024-12-10T15:31:36,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:31:36,785 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb3c78bb-d10d-461e-9a2e-2952e55b093c'
2024-12-10T15:31:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:31:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb3c78bb-d10d-461e-9a2e-2952e55b093c"

2024-12-10T15:31:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2f8d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,44557,-1]
2024-12-10T15:31:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:31:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:31:36,789 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:31:36,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af1c730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:31:36,791 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:31:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,42517,1733844695553, seqNum=-1]
2024-12-10T15:31:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:31:36,795 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:31:36,797 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,44557,1733844695385
2024-12-10T15:31:36,797 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:36,801 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:31:36,818 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:31:36,819 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:31:36,820 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38907
2024-12-10T15:31:36,821 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38907 connecting to ZooKeeper ensemble=127.0.0.1:56326
2024-12-10T15:31:36,821 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:36,823 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:31:36,844 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389070x0, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:31:36,844 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-10T15:31:36,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38907-0x100105a80090002 connected
2024-12-10T15:31:36,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-10T15:31:36,845 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:31:36,846 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:31:36,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:31:36,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:31:36,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38907
2024-12-10T15:31:36,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38907
2024-12-10T15:31:36,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38907
2024-12-10T15:31:36,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38907
2024-12-10T15:31:36,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38907
2024-12-10T15:31:36,851 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(746): ClusterId : cb3c78bb-d10d-461e-9a2e-2952e55b093c
2024-12-10T15:31:36,851 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:31:36,860 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:31:36,860 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:31:36,871 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:31:36,871 DEBUG [RS:1;d6bcd7308055:38907 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4737b6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:31:36,884 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d6bcd7308055:38907
2024-12-10T15:31:36,884 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:31:36,884 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:31:36,884 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:31:36,885 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,44557,1733844695385 with port=38907, startcode=1733844696818
2024-12-10T15:31:36,885 DEBUG [RS:1;d6bcd7308055:38907 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:31:36,889 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:31:36,889 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44557 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,890 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44557 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,891 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165
2024-12-10T15:31:36,891 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34973
2024-12-10T15:31:36,891 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:31:36,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:31:36,902 DEBUG [RS:1;d6bcd7308055:38907 {}] zookeeper.ZKUtil(111): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,902 WARN  [RS:1;d6bcd7308055:38907 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:31:36,902 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,38907,1733844696818]
2024-12-10T15:31:36,902 INFO  [RS:1;d6bcd7308055:38907 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:31:36,902 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,906 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:31:36,908 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:31:36,908 INFO  [RS:1;d6bcd7308055:38907 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:31:36,908 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,908 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:31:36,909 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:31:36,909 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:31:36,910 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:36,911 DEBUG [RS:1;d6bcd7308055:38907 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,911 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,38907,1733844696818-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:31:36,927 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:31:36,927 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,38907,1733844696818-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,927 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,927 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.Replication(171): d6bcd7308055,38907,1733844696818 started
2024-12-10T15:31:36,943 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:31:36,943 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,38907,1733844696818, RpcServer on d6bcd7308055/172.17.0.2:38907, sessionid=0x100105a80090002
2024-12-10T15:31:36,943 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:31:36,943 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;d6bcd7308055:38907,5,FailOnTimeoutGroup]
2024-12-10T15:31:36,943 DEBUG [RS:1;d6bcd7308055:38907 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,943 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,38907,1733844696818'
2024-12-10T15:31:36,943 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:31:36,944 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:31:36,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,38907,1733844696818
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,38907,1733844696818'
2024-12-10T15:31:36,944 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:31:36,945 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:31:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is d6bcd7308055,44557,1733844695385
2024-12-10T15:31:36,945 DEBUG [RS:1;d6bcd7308055:38907 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:31:36,945 INFO  [RS:1;d6bcd7308055:38907 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:31:36,945 INFO  [RS:1;d6bcd7308055:38907 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:31:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29e99e4f
2024-12-10T15:31:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T15:31:36,947 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T15:31:36,948 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T15:31:36,948 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T15:31:36,948 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:31:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T15:31:36,951 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T15:31:36,951 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:36,951 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-10T15:31:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:31:36,953 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T15:31:36,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741835_1011 (size=393)
2024-12-10T15:31:36,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741835_1011 (size=393)
2024-12-10T15:31:37,047 INFO  [RS:1;d6bcd7308055:38907 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C38907%2C1733844696818, suffix=, logDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818, archiveDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs, maxLogs=32
2024-12-10T15:31:37,048 INFO  [RS:1;d6bcd7308055:38907 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844697048
2024-12-10T15:31:37,055 INFO  [RS:1;d6bcd7308055:38907 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048
2024-12-10T15:31:37,057 DEBUG [RS:1;d6bcd7308055:38907 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46599:46599),(127.0.0.1/127.0.0.1:44155:44155)]
2024-12-10T15:31:37,362 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 041d980a2e6701215998d04be5fa9aed, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165
2024-12-10T15:31:37,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33249 is added to blk_1073741837_1013 (size=76)
2024-12-10T15:31:37,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36565 is added to blk_1073741837_1013 (size=76)
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 041d980a2e6701215998d04be5fa9aed, disabling compactions & flushes
2024-12-10T15:31:37,391 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed. after waiting 0 ms
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,391 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,391 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 041d980a2e6701215998d04be5fa9aed:
Waiting for close lock at 1733844697391Disabling compacts and flushes for region at 1733844697391Disabling writes for close at 1733844697391Writing region close event to WAL at 1733844697391Closed at 1733844697391
2024-12-10T15:31:37,393 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T15:31:37,393 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733844697393"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844697393"}]},"ts":"1733844697393"}
2024-12-10T15:31:37,396 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T15:31:37,397 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T15:31:37,397 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844697397"}]},"ts":"1733844697397"}
2024-12-10T15:31:37,400 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-10T15:31:37,403 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d6bcd7308055=0} racks are {/default-rack=0}
2024-12-10T15:31:37,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-10T15:31:37,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-10T15:31:37,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-10T15:31:37,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-10T15:31:37,407 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-10T15:31:37,407 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-10T15:31:37,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-10T15:31:37,409 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=041d980a2e6701215998d04be5fa9aed, ASSIGN}]
2024-12-10T15:31:37,410 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=041d980a2e6701215998d04be5fa9aed, ASSIGN
2024-12-10T15:31:37,411 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=041d980a2e6701215998d04be5fa9aed, ASSIGN; state=OFFLINE, location=d6bcd7308055,38907,1733844696818; forceNewPlan=false, retain=false
2024-12-10T15:31:37,562 INFO  [d6bcd7308055:44557 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-10T15:31:37,563 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=041d980a2e6701215998d04be5fa9aed, regionState=OPENING, regionLocation=d6bcd7308055,38907,1733844696818
2024-12-10T15:31:37,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=041d980a2e6701215998d04be5fa9aed, ASSIGN because future has completed
2024-12-10T15:31:37,568 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041d980a2e6701215998d04be5fa9aed, server=d6bcd7308055,38907,1733844696818}]
2024-12-10T15:31:37,723 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:31:37,725 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59069, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:31:37,730 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,731 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 041d980a2e6701215998d04be5fa9aed, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:31:37,732 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,732 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:31:37,732 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,732 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,734 INFO  [StoreOpener-041d980a2e6701215998d04be5fa9aed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 041d980a2e6701215998d04be5fa9aed 
2024-12-10T15:31:37,736 INFO  [StoreOpener-041d980a2e6701215998d04be5fa9aed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041d980a2e6701215998d04be5fa9aed columnFamilyName info
2024-12-10T15:31:37,736 DEBUG [StoreOpener-041d980a2e6701215998d04be5fa9aed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:31:37,737 INFO  [StoreOpener-041d980a2e6701215998d04be5fa9aed-1 {}] regionserver.HStore(327): Store=041d980a2e6701215998d04be5fa9aed/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:31:37,737 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,738 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,739 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,739 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,739 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,742 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,744 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:31:37,745 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 041d980a2e6701215998d04be5fa9aed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743021, jitterRate=-0.055200785398483276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:31:37,745 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:37,746 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 041d980a2e6701215998d04be5fa9aed:
Running coprocessor pre-open hook at 1733844697732Writing region info on filesystem at 1733844697732Initializing all the Stores at 1733844697734 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844697734Cleaning up temporary data from old regions at 1733844697739 (+5 ms)Running coprocessor post-open hooks at 1733844697745 (+6 ms)Region opened successfully at 1733844697746 (+1 ms)
2024-12-10T15:31:37,747 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed., pid=6, masterSystemTime=1733844697723
2024-12-10T15:31:37,750 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,750 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:37,751 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=041d980a2e6701215998d04be5fa9aed, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,38907,1733844696818
2024-12-10T15:31:37,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041d980a2e6701215998d04be5fa9aed, server=d6bcd7308055,38907,1733844696818 because future has completed
2024-12-10T15:31:37,758 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T15:31:37,758 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 041d980a2e6701215998d04be5fa9aed, server=d6bcd7308055,38907,1733844696818 in 187 msec
2024-12-10T15:31:37,762 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T15:31:37,762 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=041d980a2e6701215998d04be5fa9aed, ASSIGN in 349 msec
2024-12-10T15:31:37,763 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T15:31:37,763 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844697763"}]},"ts":"1733844697763"}
2024-12-10T15:31:37,766 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-10T15:31:37,767 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T15:31:37,770 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 819 msec
2024-12-10T15:31:38,176 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:31:38,180 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:38,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:38,205 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:38,206 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:41,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:31:41,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T15:31:41,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T15:31:41,988 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-10T15:31:41,989 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:31:41,989 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T15:31:43,234 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:31:43,236 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:43,253 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:43,255 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:43,255 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:31:43,263 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-10T15:31:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44557 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:31:47,040 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-10T15:31:47,040 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-10T15:31:47,045 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T15:31:47,045 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:47,059 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:47,062 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:47,063 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:47,063 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:47,063 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:31:47,063 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4730afc2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:47,064 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a47bf3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:47,169 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c316933{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-46105-hadoop-hdfs-3_4_1-tests_jar-_-any-9337468413859162070/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:47,170 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fcb7a1e{HTTP/1.1, (http/1.1)}{localhost:46105}
2024-12-10T15:31:47,170 INFO  [Time-limited test {}] server.Server(415): Started @121761ms
2024-12-10T15:31:47,171 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:47,205 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:47,209 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:47,210 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:47,210 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:47,210 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:31:47,210 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cf6018f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:47,211 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79cceacd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:47,318 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f941cb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-46287-hadoop-hdfs-3_4_1-tests_jar-_-any-8906601454448059862/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:47,318 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bbbad3b{HTTP/1.1, (http/1.1)}{localhost:46287}
2024-12-10T15:31:47,318 INFO  [Time-limited test {}] server.Server(415): Started @121909ms
2024-12-10T15:31:47,319 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:47,358 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:31:47,361 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:31:47,362 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:31:47,362 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:31:47,362 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:31:47,363 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c20dc85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:31:47,363 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b031ac9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:31:47,471 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@73a113a3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-38403-hadoop-hdfs-3_4_1-tests_jar-_-any-1704218505940360233/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:47,471 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b2bb50{HTTP/1.1, (http/1.1)}{localhost:38403}
2024-12-10T15:31:47,471 INFO  [Time-limited test {}] server.Server(415): Started @122062ms
2024-12-10T15:31:47,473 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:31:48,461 WARN  [Thread-866 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,461 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,482 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:48,484 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97bf230e26abc9d with lease ID 0xa3daa31765999bdb: Processing first storage report for DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b from datanode DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,484 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97bf230e26abc9d with lease ID 0xa3daa31765999bdb: from storage DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b node DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:48,484 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97bf230e26abc9d with lease ID 0xa3daa31765999bdb: Processing first storage report for DS-58b1f1e8-1d37-446c-ab4d-2a946d9d212f from datanode DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,484 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97bf230e26abc9d with lease ID 0xa3daa31765999bdb: from storage DS-58b1f1e8-1d37-446c-ab4d-2a946d9d212f node DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:48,726 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data7/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,726 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data8/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,744 WARN  [Thread-830 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:48,747 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa080ef5f282636f3 with lease ID 0xa3daa31765999bdc: Processing first storage report for DS-1048c015-8f95-4b32-83e8-3e22fc4c7325 from datanode DatanodeRegistration(127.0.0.1:39139, datanodeUuid=2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01, infoPort=42785, infoSecurePort=0, ipcPort=40593, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,747 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa080ef5f282636f3 with lease ID 0xa3daa31765999bdc: from storage DS-1048c015-8f95-4b32-83e8-3e22fc4c7325 node DatanodeRegistration(127.0.0.1:39139, datanodeUuid=2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01, infoPort=42785, infoSecurePort=0, ipcPort=40593, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:48,747 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa080ef5f282636f3 with lease ID 0xa3daa31765999bdc: Processing first storage report for DS-64dc3ec8-76e5-45c2-887e-d212499ef279 from datanode DatanodeRegistration(127.0.0.1:39139, datanodeUuid=2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01, infoPort=42785, infoSecurePort=0, ipcPort=40593, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,747 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa080ef5f282636f3 with lease ID 0xa3daa31765999bdc: from storage DS-64dc3ec8-76e5-45c2-887e-d212499ef279 node DatanodeRegistration(127.0.0.1:39139, datanodeUuid=2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01, infoPort=42785, infoSecurePort=0, ipcPort=40593, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:48,889 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data10/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,889 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data9/current/BP-1301583617-172.17.0.2-1733844693229/current, will proceed with Du for space computation calculation, 
2024-12-10T15:31:48,921 WARN  [Thread-852 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:31:48,924 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa6916fa31857853 with lease ID 0xa3daa31765999bdd: Processing first storage report for DS-2be716e5-9135-46a0-8cf1-b768e1cebef1 from datanode DatanodeRegistration(127.0.0.1:36559, datanodeUuid=48364f44-6e9e-49d1-9c25-7e732ff2052b, infoPort=41339, infoSecurePort=0, ipcPort=40439, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,924 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa6916fa31857853 with lease ID 0xa3daa31765999bdd: from storage DS-2be716e5-9135-46a0-8cf1-b768e1cebef1 node DatanodeRegistration(127.0.0.1:36559, datanodeUuid=48364f44-6e9e-49d1-9c25-7e732ff2052b, infoPort=41339, infoSecurePort=0, ipcPort=40439, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:48,924 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa6916fa31857853 with lease ID 0xa3daa31765999bdd: Processing first storage report for DS-629baa21-8dbb-4df8-b15e-88f0afb51f49 from datanode DatanodeRegistration(127.0.0.1:36559, datanodeUuid=48364f44-6e9e-49d1-9c25-7e732ff2052b, infoPort=41339, infoSecurePort=0, ipcPort=40439, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229)
2024-12-10T15:31:48,924 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa6916fa31857853 with lease ID 0xa3daa31765999bdd: from storage DS-629baa21-8dbb-4df8-b15e-88f0afb51f49 node DatanodeRegistration(127.0.0.1:36559, datanodeUuid=48364f44-6e9e-49d1-9c25-7e732ff2052b, infoPort=41339, infoSecurePort=0, ipcPort=40439, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:31:49,025 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,025 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,025 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,025 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:49,025 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:49,026 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:49,025 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,026 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:51934 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51934 dst: /127.0.0.1:33249
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:43388 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43388 dst: /127.0.0.1:36565
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,027 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:49,026 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:51894 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51894 dst: /127.0.0.1:33249
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:43422 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43422 dst: /127.0.0.1:36565
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:51954 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:33249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51954 dst: /127.0.0.1:33249
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:43434 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43434 dst: /127.0.0.1:36565
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,029 WARN  [PacketResponder: BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33249] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:43416 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43416 dst: /127.0.0.1:36565
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,031 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:51922 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51922 dst: /127.0.0.1:33249
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,038 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75e3c7fe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:49,040 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a8aaf72{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:49,040 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:49,040 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3351669c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:49,040 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76a89480{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:49,049 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:49,049 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:49,049 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 48d75f3e-a9c8-40d0-a309-761ae103fdcf) service to localhost/127.0.0.1:34973
2024-12-10T15:31:49,049 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:49,050 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data3/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:49,050 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data4/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:49,050 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:49,051 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741836_1012
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,052 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:36392 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36392 dst: /127.0.0.1:36565
java.nio.channels.ClosedChannelException: null
	at sun.nio.ch.SocketChannelImpl.ensureOpenAndConnected(SocketChannelImpl.java:215) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:527) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,053 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,053 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:36382 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36382 dst: /127.0.0.1:36565
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,061 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,062 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:36372 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36372 dst: /127.0.0.1:36565
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:36388 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36565:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36388 dst: /127.0.0.1:36565
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:49,067 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 24d0e157-4045-4a75-a330-616f0c3e7300) service to localhost/127.0.0.1:34973
2024-12-10T15:31:49,067 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data1/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:49,068 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data2/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:49,069 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741834_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,076 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6256f4f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:49,077 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@792ad53f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:49,077 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:49,077 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24f9668a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:49,077 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6586e5e2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:49,079 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed., hostname=d6bcd7308055,38907,1733844696818, seqNum=2]
2024-12-10T15:31:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:31:49,086 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:31:49,088 ERROR [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165-prefix:d6bcd7308055,38907,1733844696818 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,088 WARN  [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165-prefix:d6bcd7308055,38907,1733844696818 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,088 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C38907%2C1733844696818:(num 1733844697048) roll requested
2024-12-10T15:31:49,088 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844709088
2024-12-10T15:31:49,092 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,092 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:49,092 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741838_1018
2024-12-10T15:31:49,095 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:49,100 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,100 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:49,100 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741839_1019
2024-12-10T15:31:49,102 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:49,125 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:49,125 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:49,125 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:49,125 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:49,126 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:49,126 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088
2024-12-10T15:31:49,126 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,126 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:49,128 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-10T15:31:49,128 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-10T15:31:49,128 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048
2024-12-10T15:31:49,132 WARN  [IPC Server handler 4 on default port 34973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741836_1012
2024-12-10T15:31:49,136 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 after 6ms
2024-12-10T15:31:49,141 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41339:41339),(127.0.0.1/127.0.0.1:42785:42785)]
2024-12-10T15:31:49,141 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:49,677 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:50,129 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:50,129 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:51,143 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088
2024-12-10T15:31:51,143 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:51,144 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:51,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:51476 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:39139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51476 dst: /127.0.0.1:39139
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:51,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:52822 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:36559:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52822 dst: /127.0.0.1:36559
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:51,183 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@73a113a3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:51,184 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b2bb50{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:51,184 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:51,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b031ac9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:51,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c20dc85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:51,185 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:51,185 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:51,185 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 48364f44-6e9e-49d1-9c25-7e732ff2052b) service to localhost/127.0.0.1:34973
2024-12-10T15:31:51,185 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:51,186 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data9/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:51,186 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data10/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:51,186 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:51,677 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:52,129 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:52,129 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:53,137 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 after 4009ms
2024-12-10T15:31:53,142 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]
2024-12-10T15:31:53,142 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C38907%2C1733844696818:(num 1733844709088) roll requested
2024-12-10T15:31:53,142 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844713142
2024-12-10T15:31:53,145 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:53,145 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:53,145 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741841_1023
2024-12-10T15:31:53,146 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:53,147 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:53,147 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:53,147 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741842_1024
2024-12-10T15:31:53,148 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:53,150 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36565
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:53,150 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45652 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data8]'}, localName='127.0.0.1:39139', datanodeUuid='2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025 to mirror 127.0.0.1:36565
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:53,151 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:53,151 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025
2024-12-10T15:31:53,151 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45652 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T15:31:53,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45652 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45652 dst: /127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:53,151 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:53,158 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:53,158 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:53,158 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:53,158 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:53,158 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:53,158 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844713142
2024-12-10T15:31:53,160 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40659:40659),(127.0.0.1/127.0.0.1:42785:42785)]
2024-12-10T15:31:53,160 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:53,160 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088 is not closed yet, will try archiving it next time
2024-12-10T15:31:53,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39139 is added to blk_1073741840_1022 (size=2431)
2024-12-10T15:31:53,191 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T15:31:53,562 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:53,677 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:54,130 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:54,130 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,195 WARN  [ResponseProcessor for block BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026
java.io.IOException: Bad response ERROR for BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026 from datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,196 WARN  [DataStreamer for file /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844713142 block BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:55,196 WARN  [PacketResponder: BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39139] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,196 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39796 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39796 dst: /127.0.0.1:33755
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,196 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45658 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:39139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45658 dst: /127.0.0.1:39139
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,240 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f941cb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:31:55,240 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bbbad3b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:31:55,240 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:31:55,240 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79cceacd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:31:55,240 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cf6018f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:31:55,242 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:31:55,242 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:31:55,242 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 2e8b6afd-a1d8-4346-bc8c-5b3b3e057c01) service to localhost/127.0.0.1:34973
2024-12-10T15:31:55,242 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:31:55,242 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data7/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:55,242 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data8/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:31:55,243 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:31:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:55,252 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:31:55,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/a627aee5a4c54691b0c722c4d780424e is 1080, key is row0002/info:/1733844711187/Put/seqid=0
2024-12-10T15:31:55,273 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,273 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:55,273 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741845_1028
2024-12-10T15:31:55,274 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:55,276 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36559
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39804 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029 to mirror 127.0.0.1:36559
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,276 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:55,276 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029
2024-12-10T15:31:55,276 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39804 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:31:55,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39804 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39804 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,277 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:55,278 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,279 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:55,279 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741847_1030
2024-12-10T15:31:55,279 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:55,281 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,281 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:55,281 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741848_1031
2024-12-10T15:31:55,281 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:55,282 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:55,282 WARN  [IPC Server handler 4 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:55,282 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:55,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741849_1032 (size=10347)
2024-12-10T15:31:55,678 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,687 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/a627aee5a4c54691b0c722c4d780424e
2024-12-10T15:31:55,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/a627aee5a4c54691b0c722c4d780424e as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e
2024-12-10T15:31:55,703 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e, entries=5, sequenceid=11, filesize=10.1 K
2024-12-10T15:31:55,704 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 041d980a2e6701215998d04be5fa9aed in 451ms, sequenceid=11, compaction requested=false
2024-12-10T15:31:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:31:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:55,880 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-10T15:31:55,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/b8afeae658274f7185e0948caea1451b is 1080, key is row0007/info:/1733844715254/Put/seqid=0
2024-12-10T15:31:55,886 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,886 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:55,886 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741850_1033
2024-12-10T15:31:55,887 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:55,888 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,889 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:55,889 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741851_1034
2024-12-10T15:31:55,889 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:55,892 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39814 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,892 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:55,892 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035
2024-12-10T15:31:55,892 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39814 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:31:55,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39814 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39814 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,893 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:55,895 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36559
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:55,895 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39824 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036 to mirror 127.0.0.1:36559
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,895 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:55,896 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036
2024-12-10T15:31:55,896 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39824 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:31:55,896 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39824 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39824 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:55,896 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:55,897 WARN  [IPC Server handler 3 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:55,897 WARN  [IPC Server handler 3 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:55,897 WARN  [IPC Server handler 3 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:55,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741854_1037 (size=12506)
2024-12-10T15:31:56,130 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:56,130 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:56,302 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/b8afeae658274f7185e0948caea1451b
2024-12-10T15:31:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/b8afeae658274f7185e0948caea1451b as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b
2024-12-10T15:31:56,319 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b, entries=7, sequenceid=24, filesize=12.2 K
2024-12-10T15:31:56,320 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 041d980a2e6701215998d04be5fa9aed in 440ms, sequenceid=24, compaction requested=false
2024-12-10T15:31:56,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:31:56,321 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-10T15:31:56,321 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:56,321 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b because midkey is the same as first or last row
2024-12-10T15:31:57,160 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]
2024-12-10T15:31:57,161 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C38907%2C1733844696818:(num 1733844713142) roll requested
2024-12-10T15:31:57,161 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844717161
2024-12-10T15:31:57,165 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,165 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39846 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,165 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:57,165 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39846 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T15:31:57,165 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038
2024-12-10T15:31:57,165 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39846 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39846 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,166 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:57,167 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,167 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:57,167 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741856_1039
2024-12-10T15:31:57,168 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:57,169 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,169 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:57,169 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741857_1040
2024-12-10T15:31:57,170 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:57,172 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36565
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39860 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041 to mirror 127.0.0.1:36565
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,172 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:57,173 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041
2024-12-10T15:31:57,173 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39860 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T15:31:57,173 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39860 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39860 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,174 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:57,174 WARN  [IPC Server handler 2 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:57,175 WARN  [IPC Server handler 2 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:57,175 WARN  [IPC Server handler 2 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:57,182 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:57,182 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:57,182 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:57,182 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:57,182 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:57,183 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844713142 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844717161
2024-12-10T15:31:57,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741844_1027 (size=25992)
2024-12-10T15:31:57,189 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40659:40659)]
2024-12-10T15:31:57,189 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:57,189 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844713142 is not closed yet, will try archiving it next time
2024-12-10T15:31:57,190 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844709088 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs/d6bcd7308055%2C38907%2C1733844696818.1733844709088
2024-12-10T15:31:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:57,300 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T15:31:57,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/311601d2224e4065a1fc6589c42f509e is 1079, key is tmprow/info:/1733844717299/Put/seqid=0
2024-12-10T15:31:57,308 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,308 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:57,308 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741860_1043
2024-12-10T15:31:57,309 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:57,310 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,310 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:57,310 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741861_1044
2024-12-10T15:31:57,311 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:57,312 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,313 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:57,313 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741862_1045
2024-12-10T15:31:57,313 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:57,315 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,315 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:57,315 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741863_1046
2024-12-10T15:31:57,316 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:57,316 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:57,316 WARN  [IPC Server handler 4 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:57,317 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:57,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741864_1047 (size=6027)
2024-12-10T15:31:57,585 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:57,678 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,720 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/311601d2224e4065a1fc6589c42f509e
2024-12-10T15:31:57,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/311601d2224e4065a1fc6589c42f509e as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e
2024-12-10T15:31:57,738 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e, entries=1, sequenceid=34, filesize=5.9 K
2024-12-10T15:31:57,739 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 041d980a2e6701215998d04be5fa9aed in 439ms, sequenceid=34, compaction requested=true
2024-12-10T15:31:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:31:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-10T15:31:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b because midkey is the same as first or last row
2024-12-10T15:31:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041d980a2e6701215998d04be5fa9aed:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:31:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:31:57,740 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:31:57,742 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:31:57,742 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1541): 041d980a2e6701215998d04be5fa9aed/info is initiating minor compaction (all files)
2024-12-10T15:31:57,742 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041d980a2e6701215998d04be5fa9aed/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:31:57,742 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e] into tmpdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp, totalSize=28.2 K
2024-12-10T15:31:57,743 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting a627aee5a4c54691b0c722c4d780424e, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733844711187
2024-12-10T15:31:57,743 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8afeae658274f7185e0948caea1451b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733844715254
2024-12-10T15:31:57,744 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting 311601d2224e4065a1fc6589c42f509e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733844717299
2024-12-10T15:31:57,758 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041d980a2e6701215998d04be5fa9aed#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:31:57,759 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/31c0edc600e4414888c1ecf3a8048c9a is 1080, key is row0002/info:/1733844711187/Put/seqid=0
2024-12-10T15:31:57,761 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36559
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39908 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048 to mirror 127.0.0.1:36559
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,761 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:57,761 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048
2024-12-10T15:31:57,761 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39908 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:31:57,762 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39908 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39908 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:57,762 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:57,763 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,763 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:57,763 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741866_1049
2024-12-10T15:31:57,764 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:57,765 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,765 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:57,765 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741867_1050
2024-12-10T15:31:57,766 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:57,767 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:57,767 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:57,768 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741868_1051
2024-12-10T15:31:57,768 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:57,769 WARN  [IPC Server handler 2 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:57,769 WARN  [IPC Server handler 2 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:57,769 WARN  [IPC Server handler 2 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:57,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741869_1052 (size=17994)
2024-12-10T15:31:58,130 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,131 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,184 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/31c0edc600e4414888c1ecf3a8048c9a as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a
2024-12-10T15:31:58,195 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041d980a2e6701215998d04be5fa9aed/info of 041d980a2e6701215998d04be5fa9aed into 31c0edc600e4414888c1ecf3a8048c9a(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:31:58,195 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed., storeName=041d980a2e6701215998d04be5fa9aed/info, priority=13, startTime=1733844717740; duration=0sec
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a because midkey is the same as first or last row
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a because midkey is the same as first or last row
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a because midkey is the same as first or last row
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:31:58,195 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041d980a2e6701215998d04be5fa9aed:info
2024-12-10T15:31:58,501 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@34f76229[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741849_1032 to 127.0.0.1:36559 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:58,501 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2325b025[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741854_1037 to 127.0.0.1:33249 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:31:58,719 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T15:31:58,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f83f5783d996405193f2e86276bad8fc is 1079, key is tmprow/info:/1733844718718/Put/seqid=0
2024-12-10T15:31:58,728 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,728 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:58,728 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741870_1053
2024-12-10T15:31:58,729 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:58,730 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,730 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:58,730 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741871_1054
2024-12-10T15:31:58,731 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:58,732 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,732 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:58,733 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741872_1055
2024-12-10T15:31:58,733 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:58,734 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:58,735 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:58,735 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741873_1056
2024-12-10T15:31:58,735 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:58,736 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:58,736 WARN  [IPC Server handler 4 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:58,736 WARN  [IPC Server handler 4 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:58,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741874_1057 (size=6027)
2024-12-10T15:31:59,140 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f83f5783d996405193f2e86276bad8fc
2024-12-10T15:31:59,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f83f5783d996405193f2e86276bad8fc as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc
2024-12-10T15:31:59,156 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc, entries=1, sequenceid=45, filesize=5.9 K
2024-12-10T15:31:59,157 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 041d980a2e6701215998d04be5fa9aed in 438ms, sequenceid=45, compaction requested=false
2024-12-10T15:31:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:31:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-10T15:31:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:31:59,157 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a because midkey is the same as first or last row
2024-12-10T15:31:59,190 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]
2024-12-10T15:31:59,190 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C38907%2C1733844696818:(num 1733844717161) roll requested
2024-12-10T15:31:59,190 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844719190
2024-12-10T15:31:59,193 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:59,193 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:31:59,193 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741875_1058
2024-12-10T15:31:59,194 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:31:59,195 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:59,195 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:31:59,195 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741876_1059
2024-12-10T15:31:59,196 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:31:59,197 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:59,197 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:31:59,198 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741877_1060
2024-12-10T15:31:59,198 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:31:59,200 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36559
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:31:59,200 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39930 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061 to mirror 127.0.0.1:36559
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:59,201 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:31:59,201 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39930 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061] {}] datanode.BlockReceiver(316): Block 1073741878 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T15:31:59,201 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061
2024-12-10T15:31:59,201 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:39930 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741878_1061] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39930 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:59,201 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:31:59,202 WARN  [IPC Server handler 0 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:31:59,202 WARN  [IPC Server handler 0 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:31:59,202 WARN  [IPC Server handler 0 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:31:59,205 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:59,205 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:59,205 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:59,205 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:59,205 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:31:59,206 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844717161 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844719190
2024-12-10T15:31:59,206 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40659:40659)]
2024-12-10T15:31:59,206 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:59,207 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844717161 is not closed yet, will try archiving it next time
2024-12-10T15:31:59,207 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844713142 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs/d6bcd7308055%2C38907%2C1733844696818.1733844713142
2024-12-10T15:31:59,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741859_1042 (size=13591)
2024-12-10T15:31:59,490 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@34f76229[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741864_1047 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:59,490 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2325b025[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741844_1027 to 127.0.0.1:36559 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:31:59,608 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 is not closed yet, will try archiving it next time
2024-12-10T15:31:59,679 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,131 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,131 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:32:00,141 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T15:32:00,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/18af61b0629a4e3da16262c4ac2ffc92 is 1079, key is tmprow/info:/1733844720140/Put/seqid=0
2024-12-10T15:32:00,148 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,149 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:32:00,149 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741880_1063
2024-12-10T15:32:00,149 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:32:00,151 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,151 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:32:00,151 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741881_1064
2024-12-10T15:32:00,152 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:32:00,153 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,153 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:32:00,153 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741882_1065
2024-12-10T15:32:00,154 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:32:00,156 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45546 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:00,156 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:00,156 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066
2024-12-10T15:32:00,156 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45546 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:32:00,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45546 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741883_1066] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45546 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:00,157 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:00,158 WARN  [IPC Server handler 0 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:32:00,158 WARN  [IPC Server handler 0 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:32:00,158 WARN  [IPC Server handler 0 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:32:00,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741884_1067 (size=6027)
2024-12-10T15:32:00,561 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/18af61b0629a4e3da16262c4ac2ffc92
2024-12-10T15:32:00,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/18af61b0629a4e3da16262c4ac2ffc92 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92
2024-12-10T15:32:00,573 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92, entries=1, sequenceid=55, filesize=5.9 K
2024-12-10T15:32:00,574 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 041d980a2e6701215998d04be5fa9aed in 433ms, sequenceid=55, compaction requested=true
2024-12-10T15:32:00,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:32:00,574 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-12-10T15:32:00,575 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:00,575 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a because midkey is the same as first or last row
2024-12-10T15:32:00,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041d980a2e6701215998d04be5fa9aed:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:32:00,575 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:32:00,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:32:00,576 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:32:00,576 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1541): 041d980a2e6701215998d04be5fa9aed/info is initiating minor compaction (all files)
2024-12-10T15:32:00,576 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041d980a2e6701215998d04be5fa9aed/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:00,576 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92] into tmpdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp, totalSize=29.3 K
2024-12-10T15:32:00,577 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c0edc600e4414888c1ecf3a8048c9a, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733844711187
2024-12-10T15:32:00,577 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83f5783d996405193f2e86276bad8fc, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733844718718
2024-12-10T15:32:00,577 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18af61b0629a4e3da16262c4ac2ffc92, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733844720140
2024-12-10T15:32:00,595 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041d980a2e6701215998d04be5fa9aed#info#compaction#24 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:32:00,595 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/b8b3a2220c214a4c804d1b72fc3e7358 is 1080, key is row0002/info:/1733844711187/Put/seqid=0
2024-12-10T15:32:00,597 WARN  [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,597 WARN  [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:00,597 WARN  [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741885_1068
2024-12-10T15:32:00,598 WARN  [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:00,599 WARN  [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,599 WARN  [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]) is bad.
2024-12-10T15:32:00,600 WARN  [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741886_1069
2024-12-10T15:32:00,600 WARN  [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33249,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]
2024-12-10T15:32:00,603 WARN  [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36565
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,603 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45564 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070 to mirror 127.0.0.1:36565
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:00,603 WARN  [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]) is bad.
2024-12-10T15:32:00,603 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45564 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:32:00,603 WARN  [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070
2024-12-10T15:32:00,603 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:45564 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741887_1070] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45564 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:00,604 WARN  [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]
2024-12-10T15:32:00,605 WARN  [Thread-976 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:00,606 WARN  [Thread-976 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]) is bad.
2024-12-10T15:32:00,606 WARN  [Thread-976 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741888_1071
2024-12-10T15:32:00,606 WARN  [Thread-976 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36559,DS-2be716e5-9135-46a0-8cf1-b768e1cebef1,DISK]
2024-12-10T15:32:00,607 WARN  [IPC Server handler 1 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T15:32:00,607 WARN  [IPC Server handler 1 on default port 34973 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T15:32:00,607 WARN  [IPC Server handler 1 on default port 34973 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T15:32:00,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741889_1072 (size=18097)
2024-12-10T15:32:01,018 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/b8b3a2220c214a4c804d1b72fc3e7358 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358
2024-12-10T15:32:01,025 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041d980a2e6701215998d04be5fa9aed/info of 041d980a2e6701215998d04be5fa9aed into b8b3a2220c214a4c804d1b72fc3e7358(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:32:01,025 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:32:01,025 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed., storeName=041d980a2e6701215998d04be5fa9aed/info, priority=13, startTime=1733844720575; duration=0sec
2024-12-10T15:32:01,025 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T15:32:01,025 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:01,025 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 because midkey is the same as first or last row
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 because midkey is the same as first or last row
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 because midkey is the same as first or last row
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:32:01,026 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041d980a2e6701215998d04be5fa9aed:info
2024-12-10T15:32:01,207 WARN  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-10T15:32:01,369 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:01,372 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:01,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:01,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:01,373 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:32:01,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1712bc01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:01,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5daabe6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:01,470 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3948d06{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/java.io.tmpdir/jetty-localhost-43927-hadoop-hdfs-3_4_1-tests_jar-_-any-1957994924210828478/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:01,471 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44c1c692{HTTP/1.1, (http/1.1)}{localhost:43927}
2024-12-10T15:32:01,471 INFO  [Time-limited test {}] server.Server(415): Started @136061ms
2024-12-10T15:32:01,472 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:01,489 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2325b025[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741874_1057 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:01,489 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@34f76229[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741869_1052 to 127.0.0.1:36565 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:01,679 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:01,891 WARN  [Thread-995 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:01,900 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf806a573dbe3879e with lease ID 0xa3daa31765999bde: from storage DS-8bbd286c-9234-4134-a894-534f3213518a node DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 6, hasStaleStorage: false, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-10T15:32:01,901 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf806a573dbe3879e with lease ID 0xa3daa31765999bde: from storage DS-b0b9f480-fb1c-4ee7-9c13-c9827e768128 node DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:02,131 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:02,132 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:02,490 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2325b025[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741884_1067 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:02,490 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@34f76229[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741859_1042 to 127.0.0.1:36565 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:03,679 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:04,132 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:04,132 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:04,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741889_1072 (size=18097)
2024-12-10T15:32:05,359 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:32:05,680 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,916 ERROR [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData-prefix:d6bcd7308055,44557,1733844695385 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,917 WARN  [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData-prefix:d6bcd7308055,44557,1733844695385 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,917 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C44557%2C1733844695385:(num 1733844695698) roll requested
2024-12-10T15:32:05,917 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44557%2C1733844695385.1733844725917
2024-12-10T15:32:05,921 WARN  [Thread-1016 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1073
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,921 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:33038 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data4]'}, localName='127.0.0.1:35247', datanodeUuid='48d75f3e-a9c8-40d0-a309-761ae103fdcf', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:05,921 WARN  [Thread-1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35247,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:05,921 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:33038 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T15:32:05,921 WARN  [Thread-1016 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073
2024-12-10T15:32:05,921 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-783470468_22 at /127.0.0.1:33038 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741890_1073] {}] datanode.DataXceiver(331): 127.0.0.1:35247:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33038 dst: /127.0.0.1:35247
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:05,922 WARN  [Thread-1016 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:05,928 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:05,928 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:05,928 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:05,928 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:05,928 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:05,929 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 with entries=54, filesize=26.65 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844725917
2024-12-10T15:32:05,929 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,929 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:05,929 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
2024-12-10T15:32:05,930 WARN  [IPC Server handler 0 on default port 34973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741830_1014
2024-12-10T15:32:05,930 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 after 1ms
2024-12-10T15:32:05,934 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42547:42547),(127.0.0.1/127.0.0.1:40659:40659)]
2024-12-10T15:32:05,934 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 is not closed yet, will try archiving it next time
2024-12-10T15:32:06,133 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:06,133 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:08,133 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:08,133 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:09,896 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1ccf4107[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741835_1011 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:09,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741837_1013 (size=76)
2024-12-10T15:32:09,931 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 after 4002ms
2024-12-10T15:32:10,134 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:10,134 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:10,895 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1ccf4107[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741829_1005 to 127.0.0.1:36559 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:10,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:32:11,916 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2da8ab08 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1012, datanode=DatanodeInfoWithStorage[127.0.0.1:36565,null,null])
java.net.ConnectException: Call From d6bcd7308055/172.17.0.2 to localhost:35079 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-10T15:32:11,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741836_1021 (size=455)
2024-12-10T15:32:12,134 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:12,134 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:12,155 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844697048 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs/d6bcd7308055%2C38907%2C1733844696818.1733844697048
2024-12-10T15:32:12,156 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844717161 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs/d6bcd7308055%2C38907%2C1733844696818.1733844717161
2024-12-10T15:32:12,896 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1ccf4107[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741836_1021 to 127.0.0.1:36559 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:12,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:32:14,134 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:14,135 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:14,808 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C38907%2C1733844696818.1733844734807
2024-12-10T15:32:14,816 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:14,816 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:14,816 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:14,816 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:14,816 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:14,817 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844719190 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844734807
2024-12-10T15:32:14,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40659:40659),(127.0.0.1/127.0.0.1:42547:42547)]
2024-12-10T15:32:14,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844719190 is not closed yet, will try archiving it next time
2024-12-10T15:32:14,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741879_1062 (size=12911)
2024-12-10T15:32:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:32:14,824 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T15:32:14,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/e33d35141b384891a644c79bb0c6a8f6 is 1080, key is row0013/info:/1733844734820/Put/seqid=0
2024-12-10T15:32:14,832 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:14,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:47738 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6]'}, localName='127.0.0.1:33755', datanodeUuid='0273a4e0-17eb-4e56-b28a-60eddd519f3e', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:14,832 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:14,832 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077
2024-12-10T15:32:14,832 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:47738 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:32:14,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-539990753_22 at /127.0.0.1:47738 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:33755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47738 dst: /127.0.0.1:33755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:14,833 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:14,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741894_1078 (size=8190)
2024-12-10T15:32:14,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741894_1078 (size=8190)
2024-12-10T15:32:14,839 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/e33d35141b384891a644c79bb0c6a8f6
2024-12-10T15:32:14,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/e33d35141b384891a644c79bb0c6a8f6 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6
2024-12-10T15:32:14,854 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6, entries=3, sequenceid=66, filesize=8.0 K
2024-12-10T15:32:14,855 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 041d980a2e6701215998d04be5fa9aed in 31ms, sequenceid=66, compaction requested=false
2024-12-10T15:32:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:32:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-12-10T15:32:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 because midkey is the same as first or last row
2024-12-10T15:32:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38907 {}] regionserver.HRegion(8855): Flush requested on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:32:15,051 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041d980a2e6701215998d04be5fa9aed 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-10T15:32:15,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f0a6d9ffa2e34dc681258163e44a66c5 is 1080, key is row0015/info:/1733844734825/Put/seqid=0
2024-12-10T15:32:15,060 WARN  [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,060 WARN  [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:35247,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:15,060 WARN  [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741895_1079
2024-12-10T15:32:15,061 WARN  [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:15,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741896_1080 (size=14660)
2024-12-10T15:32:15,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741896_1080 (size=14660)
2024-12-10T15:32:15,067 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f0a6d9ffa2e34dc681258163e44a66c5
2024-12-10T15:32:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/f0a6d9ffa2e34dc681258163e44a66c5 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5
2024-12-10T15:32:15,081 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5, entries=9, sequenceid=79, filesize=14.3 K
2024-12-10T15:32:15,082 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 041d980a2e6701215998d04be5fa9aed in 31ms, sequenceid=79, compaction requested=true
2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 because midkey is the same as first or last row
2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041d980a2e6701215998d04be5fa9aed:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:32:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:32:15,082 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:32:15,084 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:32:15,084 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1541): 041d980a2e6701215998d04be5fa9aed/info is initiating minor compaction (all files)
2024-12-10T15:32:15,084 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041d980a2e6701215998d04be5fa9aed/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,084 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5] into tmpdir=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp, totalSize=40.0 K
2024-12-10T15:32:15,085 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b3a2220c214a4c804d1b72fc3e7358, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733844711187
2024-12-10T15:32:15,086 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33d35141b384891a644c79bb0c6a8f6, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733844721157
2024-12-10T15:32:15,086 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0a6d9ffa2e34dc681258163e44a66c5, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733844734825
2024-12-10T15:32:15,102 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041d980a2e6701215998d04be5fa9aed#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:32:15,102 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/e19577dbfb0446b8a8d70ead762bb665 is 1080, key is row0002/info:/1733844711187/Put/seqid=0
2024-12-10T15:32:15,104 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,104 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:15,104 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741897_1081
2024-12-10T15:32:15,105 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:15,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741898_1082 (size=28989)
2024-12-10T15:32:15,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741898_1082 (size=28989)
2024-12-10T15:32:15,117 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/.tmp/info/e19577dbfb0446b8a8d70ead762bb665 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e19577dbfb0446b8a8d70ead762bb665
2024-12-10T15:32:15,123 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041d980a2e6701215998d04be5fa9aed/info of 041d980a2e6701215998d04be5fa9aed into e19577dbfb0446b8a8d70ead762bb665(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:32:15,123 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041d980a2e6701215998d04be5fa9aed:

2024-12-10T15:32:15,123 INFO  [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed., storeName=041d980a2e6701215998d04be5fa9aed/info, priority=13, startTime=1733844735082; duration=0sec
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e19577dbfb0446b8a8d70ead762bb665 because midkey is the same as first or last row
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e19577dbfb0446b8a8d70ead762bb665 because midkey is the same as first or last row
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e19577dbfb0446b8a8d70ead762bb665 because midkey is the same as first or last row
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:32:15,124 DEBUG [RS:1;d6bcd7308055:38907-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041d980a2e6701215998d04be5fa9aed:info
2024-12-10T15:32:15,208 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-10T15:32:15,220 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,38907,1733844696818/d6bcd7308055%2C38907%2C1733844696818.1733844719190 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs/d6bcd7308055%2C38907%2C1733844696818.1733844719190
2024-12-10T15:32:15,252 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:32:15,252 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:32:15,252 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:32:15,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,253 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:32:15,253 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:32:15,253 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=926786715, stopped=false
2024-12-10T15:32:15,254 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,44557,1733844695385
2024-12-10T15:32:15,399 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:32:15,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:32:15,399 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:15,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:15,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:32:15,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:15,399 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:32:15,399 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:32:15,400 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:32:15,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,400 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:32:15,400 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,42517,1733844695553' *****
2024-12-10T15:32:15,400 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:32:15,400 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:32:15,400 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,38907,1733844696818' *****
2024-12-10T15:32:15,400 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:32:15,400 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:32:15,401 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:32:15,401 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:32:15,401 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:32:15,401 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:32:15,401 INFO  [RS:0;d6bcd7308055:42517 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:32:15,401 INFO  [RS:0;d6bcd7308055:42517 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:32:15,401 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,42517,1733844695553
2024-12-10T15:32:15,401 INFO  [RS:1;d6bcd7308055:38907 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:32:15,401 INFO  [RS:1;d6bcd7308055:38907 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:32:15,401 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:32:15,401 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(3091): Received CLOSE for 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:32:15,401 INFO  [RS:0;d6bcd7308055:42517 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:42517.
2024-12-10T15:32:15,402 DEBUG [RS:0;d6bcd7308055:42517 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:32:15,402 DEBUG [RS:0;d6bcd7308055:42517 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,402 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,38907,1733844696818
2024-12-10T15:32:15,402 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:32:15,402 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:32:15,402 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:32:15,402 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:32:15,402 INFO  [RS:1;d6bcd7308055:38907 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d6bcd7308055:38907.
2024-12-10T15:32:15,402 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:32:15,402 DEBUG [RS:1;d6bcd7308055:38907 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:32:15,402 DEBUG [RS:1;d6bcd7308055:38907 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,402 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T15:32:15,402 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T15:32:15,403 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1325): Online Regions={041d980a2e6701215998d04be5fa9aed=TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.}
2024-12-10T15:32:15,403 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-10T15:32:15,403 DEBUG [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1351): Waiting on 041d980a2e6701215998d04be5fa9aed
2024-12-10T15:32:15,403 DEBUG [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 041d980a2e6701215998d04be5fa9aed, disabling compactions & flushes
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:32:15,403 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,403 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed. after waiting 0 ms
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:32:15,403 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:32:15,403 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-10T15:32:15,404 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5] to archive
2024-12-10T15:32:15,404 ERROR [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165-prefix:d6bcd7308055,42517,1733844695553.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,404 WARN  [FSHLog-0-hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165-prefix:d6bcd7308055,42517,1733844695553.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,404 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,404 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C42517%2C1733844695553.meta:.meta(num 1733844696659) roll requested
2024-12-10T15:32:15,405 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C42517%2C1733844695553.meta.1733844735405.meta
2024-12-10T15:32:15,405 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:32:15,408 WARN  [Thread-1059 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,408 WARN  [Thread-1059 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741899_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:15,408 WARN  [Thread-1059 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741899_1083
2024-12-10T15:32:15,409 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/a627aee5a4c54691b0c722c4d780424e
2024-12-10T15:32:15,409 WARN  [Thread-1059 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:15,410 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8afeae658274f7185e0948caea1451b
2024-12-10T15:32:15,412 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/311601d2224e4065a1fc6589c42f509e
2024-12-10T15:32:15,412 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f83f5783d996405193f2e86276bad8fc
2024-12-10T15:32:15,413 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/18af61b0629a4e3da16262c4ac2ffc92
2024-12-10T15:32:15,414 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/b8b3a2220c214a4c804d1b72fc3e7358
2024-12-10T15:32:15,414 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/f0a6d9ffa2e34dc681258163e44a66c5
2024-12-10T15:32:15,414 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6 to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/e33d35141b384891a644c79bb0c6a8f6
2024-12-10T15:32:15,414 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,415 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,415 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,415 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,415 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,415 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844735405.meta
2024-12-10T15:32:15,415 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,416 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,416 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
2024-12-10T15:32:15,416 WARN  [IPC Server handler 2 on default port 34973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta has not been closed. Lease recovery is in progress. RecoveryId = 1085 for block blk_1073741834_1016
2024-12-10T15:32:15,416 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta after 0ms
2024-12-10T15:32:15,418 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a to hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/info/31c0edc600e4414888c1ecf3a8048c9a
2024-12-10T15:32:15,418 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d6bcd7308055:44557 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T15:32:15,419 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [a627aee5a4c54691b0c722c4d780424e=10347, b8afeae658274f7185e0948caea1451b=12506, 31c0edc600e4414888c1ecf3a8048c9a=17994, 311601d2224e4065a1fc6589c42f509e=6027, f83f5783d996405193f2e86276bad8fc=6027, b8b3a2220c214a4c804d1b72fc3e7358=18097, 18af61b0629a4e3da16262c4ac2ffc92=6027, e33d35141b384891a644c79bb0c6a8f6=8190, f0a6d9ffa2e34dc681258163e44a66c5=14660]
2024-12-10T15:32:15,425 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40659:40659),(127.0.0.1/127.0.0.1:42547:42547)]
2024-12-10T15:32:15,425 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta is not closed yet, will try archiving it next time
2024-12-10T15:32:15,429 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/default/TestLogRolling-testLogRollOnDatanodeDeath/041d980a2e6701215998d04be5fa9aed/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1
2024-12-10T15:32:15,430 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,430 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 041d980a2e6701215998d04be5fa9aed:
Waiting for close lock at 1733844735403Running coprocessor pre-close hooks at 1733844735403Disabling compacts and flushes for region at 1733844735403Disabling writes for close at 1733844735403Writing region close event to WAL at 1733844735425 (+22 ms)Running coprocessor post-close hooks at 1733844735430 (+5 ms)Closed at 1733844735430
2024-12-10T15:32:15,430 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed.
2024-12-10T15:32:15,445 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/info/d06a2d3fa5164f048ad952f3395f0aa8 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733844696948.041d980a2e6701215998d04be5fa9aed./info:regioninfo/1733844697751/Put/seqid=0
2024-12-10T15:32:15,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741901_1086 (size=7089)
2024-12-10T15:32:15,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741901_1086 (size=7089)
2024-12-10T15:32:15,451 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/info/d06a2d3fa5164f048ad952f3395f0aa8
2024-12-10T15:32:15,475 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/ns/396ce133e69c4accbef18a254ce9daa2 is 43, key is default/ns:d/1733844696744/Put/seqid=0
2024-12-10T15:32:15,478 WARN  [Thread-1072 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1087
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:51302 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data4]'}, localName='127.0.0.1:35247', datanodeUuid='48d75f3e-a9c8-40d0-a309-761ae103fdcf', xmitsInProgress=0}:Exception transferring block BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087 to mirror 127.0.0.1:39139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:15,478 WARN  [Thread-1072 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35247,DS-8bbd286c-9234-4134-a894-534f3213518a,DISK], DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:15,478 WARN  [Thread-1072 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087
2024-12-10T15:32:15,478 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:51302 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087] {}] datanode.BlockReceiver(316): Block 1073741902 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T15:32:15,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1949468320_22 at /127.0.0.1:51302 [Receiving block BP-1301583617-172.17.0.2-1733844693229:blk_1073741902_1087] {}] datanode.DataXceiver(331): 127.0.0.1:35247:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51302 dst: /127.0.0.1:35247
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:15,479 WARN  [Thread-1072 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:15,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741903_1088 (size=5153)
2024-12-10T15:32:15,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741903_1088 (size=5153)
2024-12-10T15:32:15,484 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/ns/396ce133e69c4accbef18a254ce9daa2
2024-12-10T15:32:15,506 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/table/50d22da64aaa49ca9dcc740e4243811b is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733844697763/Put/seqid=0
2024-12-10T15:32:15,508 WARN  [Thread-1079 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1089
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:15,508 WARN  [Thread-1079 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1301583617-172.17.0.2-1733844693229:blk_1073741904_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK], DatanodeInfoWithStorage[127.0.0.1:33755,DS-cbcd067d-bd09-4c9a-bc9b-f6f87a86a36b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]) is bad.
2024-12-10T15:32:15,508 WARN  [Thread-1079 {}] hdfs.DataStreamer(1850): Abandoning BP-1301583617-172.17.0.2-1733844693229:blk_1073741904_1089
2024-12-10T15:32:15,509 WARN  [Thread-1079 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39139,DS-1048c015-8f95-4b32-83e8-3e22fc4c7325,DISK]
2024-12-10T15:32:15,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741905_1090 (size=5424)
2024-12-10T15:32:15,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741905_1090 (size=5424)
2024-12-10T15:32:15,514 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/table/50d22da64aaa49ca9dcc740e4243811b
2024-12-10T15:32:15,521 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/info/d06a2d3fa5164f048ad952f3395f0aa8 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/info/d06a2d3fa5164f048ad952f3395f0aa8
2024-12-10T15:32:15,528 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/info/d06a2d3fa5164f048ad952f3395f0aa8, entries=10, sequenceid=11, filesize=6.9 K
2024-12-10T15:32:15,529 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/ns/396ce133e69c4accbef18a254ce9daa2 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/ns/396ce133e69c4accbef18a254ce9daa2
2024-12-10T15:32:15,536 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/ns/396ce133e69c4accbef18a254ce9daa2, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T15:32:15,538 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/.tmp/table/50d22da64aaa49ca9dcc740e4243811b as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/table/50d22da64aaa49ca9dcc740e4243811b
2024-12-10T15:32:15,545 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/table/50d22da64aaa49ca9dcc740e4243811b, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T15:32:15,547 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false
2024-12-10T15:32:15,551 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T15:32:15,552 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:32:15,552 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:32:15,552 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844735403Running coprocessor pre-close hooks at 1733844735403Disabling compacts and flushes for region at 1733844735403Disabling writes for close at 1733844735403Obtaining lock to block concurrent updates at 1733844735403Preparing flush snapshotting stores in 1588230740 at 1733844735403Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733844735404 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733844735429 (+25 ms)Flushing 1588230740/info: creating writer at 1733844735429Flushing 1588230740/info: appending metadata at 1733844735445 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733844735445Flushing 1588230740/ns: creating writer at 1733844735458 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733844735474 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733844735474Flushing 1588230740/table: creating writer at 1733844735490 (+16 ms)Flushing 1588230740/table: appending metadata at 1733844735505 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733844735505Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@91779f0: reopening flushed file at 1733844735520 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74d1d439: reopening flushed file at 1733844735528 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70eb10af: reopening flushed file at 1733844735536 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false at 1733844735547 (+11 ms)Writing region close event to WAL at 1733844735548 (+1 ms)Running coprocessor post-close hooks at 1733844735552 (+4 ms)Closed at 1733844735552
2024-12-10T15:32:15,552 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:32:15,603 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,42517,1733844695553; all regions closed.
2024-12-10T15:32:15,603 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,38907,1733844696818; all regions closed.
2024-12-10T15:32:15,603 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,603 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,603 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,603 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,604 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:15,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741892_1076 (size=16308)
2024-12-10T15:32:15,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741892_1076 (size=16308)
2024-12-10T15:32:15,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741900_1084 (size=825)
2024-12-10T15:32:15,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741900_1084 (size=825)
2024-12-10T15:32:15,608 DEBUG [RS:1;d6bcd7308055:38907 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs
2024-12-10T15:32:15,608 INFO  [RS:1;d6bcd7308055:38907 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C38907%2C1733844696818:(num 1733844734807)
2024-12-10T15:32:15,608 DEBUG [RS:1;d6bcd7308055:38907 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:15,608 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:32:15,608 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:32:15,609 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:32:15,609 INFO  [RS:1;d6bcd7308055:38907 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38907
2024-12-10T15:32:15,620 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,38907,1733844696818
2024-12-10T15:32:15,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:32:15,620 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:32:15,630 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,38907,1733844696818]
2024-12-10T15:32:15,640 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,38907,1733844696818 already deleted, retry=false
2024-12-10T15:32:15,641 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,38907,1733844696818 expired; onlineServers=1
2024-12-10T15:32:15,730 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:15,730 INFO  [RS:1;d6bcd7308055:38907 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:32:15,730 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38907-0x100105a80090002, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:15,730 INFO  [RS:1;d6bcd7308055:38907 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,38907,1733844696818; zookeeper connection closed.
2024-12-10T15:32:15,731 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7251c99 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7251c99
2024-12-10T15:32:15,895 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5d92efbf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35247, datanodeUuid=48d75f3e-a9c8-40d0-a309-761ae103fdcf, infoPort=42547, infoSecurePort=0, ipcPort=43685, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741827_1003 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:15,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:32:16,129 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T15:32:16,129 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T15:32:16,134 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:32:16,493 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@34f76229[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33755, datanodeUuid=0273a4e0-17eb-4e56-b28a-60eddd519f3e, infoPort=40659, infoSecurePort=0, ipcPort=34629, storageInfo=lv=-57;cid=testClusterID;nsid=1110861304;c=1733844693229):Failed to transfer BP-1301583617-172.17.0.2-1733844693229:blk_1073741879_1062 to 127.0.0.1:39139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:16,761 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T15:32:16,761 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T15:32:16,913 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:32:16,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:32:16,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:32:17,426 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:19,417 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta after 4001ms
2024-12-10T15:32:19,426 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:20,431 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,604 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-10T15:32:20,608 DEBUG [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs
2024-12-10T15:32:20,608 INFO  [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C42517%2C1733844695553.meta:.meta(num 1733844735405)
2024-12-10T15:32:20,608 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:20,608 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:20,608 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:20,608 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:20,609 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:20,609 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:20,609 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36565,DS-64f45c27-6145-4326-85b1-d289e2439498,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:20,609 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
2024-12-10T15:32:20,610 WARN  [IPC Server handler 1 on default port 34973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295 has not been closed. Lease recovery is in progress. RecoveryId = 1091 for block blk_1073741833_1015
2024-12-10T15:32:20,610 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295 after 1ms
2024-12-10T15:32:20,966 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:32:20,979 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,980 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,980 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,980 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,981 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,983 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,984 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:20,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:21,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:21,426 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-12-10T15:32:21,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T15:32:21,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:32:21,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:32:22,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:23,427 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-12-10T15:32:23,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:24,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:24,611 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295 after 4002ms
2024-12-10T15:32:25,427 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-12-10T15:32:25,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:25,609 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-10T15:32:25,611 DEBUG [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/oldWALs
2024-12-10T15:32:25,611 INFO  [RS:0;d6bcd7308055:42517 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C42517%2C1733844695553:(num 1733844696295)
2024-12-10T15:32:25,611 DEBUG [RS:0;d6bcd7308055:42517 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:25,611 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:32:25,611 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:32:25,611 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:32:25,611 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:32:25,611 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:32:25,612 INFO  [RS:0;d6bcd7308055:42517 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42517
2024-12-10T15:32:25,613 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:25,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,42517,1733844695553
2024-12-10T15:32:25,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:32:25,678 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:32:25,689 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,42517,1733844695553]
2024-12-10T15:32:25,699 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,42517,1733844695553 already deleted, retry=false
2024-12-10T15:32:25,699 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,42517,1733844695553 expired; onlineServers=0
2024-12-10T15:32:25,699 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,44557,1733844695385' *****
2024-12-10T15:32:25,699 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:32:25,699 INFO  [M:0;d6bcd7308055:44557 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:32:25,699 INFO  [M:0;d6bcd7308055:44557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:32:25,699 DEBUG [M:0;d6bcd7308055:44557 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:32:25,700 DEBUG [M:0;d6bcd7308055:44557 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:32:25,700 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:32:25,700 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844695916 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844695916,5,FailOnTimeoutGroup]
2024-12-10T15:32:25,700 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844695921 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844695921,5,FailOnTimeoutGroup]
2024-12-10T15:32:25,700 INFO  [M:0;d6bcd7308055:44557 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:32:25,700 INFO  [M:0;d6bcd7308055:44557 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:32:25,700 DEBUG [M:0;d6bcd7308055:44557 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:32:25,700 INFO  [M:0;d6bcd7308055:44557 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:32:25,700 INFO  [M:0;d6bcd7308055:44557 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:32:25,701 INFO  [M:0;d6bcd7308055:44557 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:32:25,701 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:32:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:32:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:25,710 DEBUG [M:0;d6bcd7308055:44557 {}] zookeeper.ZKUtil(347): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T15:32:25,710 WARN  [M:0;d6bcd7308055:44557 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T15:32:25,711 INFO  [M:0;d6bcd7308055:44557 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/.lastflushedseqids
2024-12-10T15:32:25,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741906_1092 (size=130)
2024-12-10T15:32:25,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741906_1092 (size=130)
2024-12-10T15:32:25,717 INFO  [M:0;d6bcd7308055:44557 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:32:25,717 INFO  [M:0;d6bcd7308055:44557 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:32:25,718 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:32:25,718 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:25,718 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:25,718 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:32:25,718 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:25,718 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.24 KB heapSize=29.47 KB
2024-12-10T15:32:25,735 DEBUG [M:0;d6bcd7308055:44557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7796b6c5eac644cea9448f7a72202008 is 82, key is hbase:meta,,1/info:regioninfo/1733844696691/Put/seqid=0
2024-12-10T15:32:25,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741907_1093 (size=5672)
2024-12-10T15:32:25,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741907_1093 (size=5672)
2024-12-10T15:32:25,741 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7796b6c5eac644cea9448f7a72202008
2024-12-10T15:32:25,763 DEBUG [M:0;d6bcd7308055:44557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f3e342074eb4301b7ee1f9083f265d3 is 773, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733844697769/Put/seqid=0
2024-12-10T15:32:25,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741908_1094 (size=6254)
2024-12-10T15:32:25,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741908_1094 (size=6254)
2024-12-10T15:32:25,772 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f3e342074eb4301b7ee1f9083f265d3
2024-12-10T15:32:25,778 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f3e342074eb4301b7ee1f9083f265d3
2024-12-10T15:32:25,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:25,789 INFO  [RS:0;d6bcd7308055:42517 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:32:25,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42517-0x100105a80090001, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:25,789 INFO  [RS:0;d6bcd7308055:42517 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,42517,1733844695553; zookeeper connection closed.
2024-12-10T15:32:25,789 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73bdc88d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73bdc88d
2024-12-10T15:32:25,789 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-10T15:32:25,793 DEBUG [M:0;d6bcd7308055:44557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f5e304e2f34584ba6f16ad43f5f58f is 69, key is d6bcd7308055,38907,1733844696818/rs:state/1733844696890/Put/seqid=0
2024-12-10T15:32:25,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741909_1095 (size=5224)
2024-12-10T15:32:25,799 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f5e304e2f34584ba6f16ad43f5f58f
2024-12-10T15:32:25,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741909_1095 (size=5224)
2024-12-10T15:32:25,819 DEBUG [M:0;d6bcd7308055:44557 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a70f91422a2a4e2ab11134b42755aa1d is 52, key is load_balancer_on/state:d/1733844696799/Put/seqid=0
2024-12-10T15:32:25,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741910_1096 (size=5056)
2024-12-10T15:32:25,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741910_1096 (size=5056)
2024-12-10T15:32:25,843 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a70f91422a2a4e2ab11134b42755aa1d
2024-12-10T15:32:25,849 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7796b6c5eac644cea9448f7a72202008 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7796b6c5eac644cea9448f7a72202008
2024-12-10T15:32:25,855 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7796b6c5eac644cea9448f7a72202008, entries=8, sequenceid=60, filesize=5.5 K
2024-12-10T15:32:25,856 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f3e342074eb4301b7ee1f9083f265d3 as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f3e342074eb4301b7ee1f9083f265d3
2024-12-10T15:32:25,862 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f3e342074eb4301b7ee1f9083f265d3
2024-12-10T15:32:25,862 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f3e342074eb4301b7ee1f9083f265d3, entries=6, sequenceid=60, filesize=6.1 K
2024-12-10T15:32:25,864 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f5e304e2f34584ba6f16ad43f5f58f as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21f5e304e2f34584ba6f16ad43f5f58f
2024-12-10T15:32:25,870 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21f5e304e2f34584ba6f16ad43f5f58f, entries=2, sequenceid=60, filesize=5.1 K
2024-12-10T15:32:25,871 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a70f91422a2a4e2ab11134b42755aa1d as hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a70f91422a2a4e2ab11134b42755aa1d
2024-12-10T15:32:25,876 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a70f91422a2a4e2ab11134b42755aa1d, entries=1, sequenceid=60, filesize=4.9 K
2024-12-10T15:32:25,878 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=60, compaction requested=false
2024-12-10T15:32:25,881 INFO  [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:25,881 DEBUG [M:0;d6bcd7308055:44557 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844745717Disabling compacts and flushes for region at 1733844745717Disabling writes for close at 1733844745718 (+1 ms)Obtaining lock to block concurrent updates at 1733844745718Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844745718Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23793, getHeapSize=30112, getOffHeapSize=0, getCellsCount=71 at 1733844745718Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844745719 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844745719Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844745735 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844745735Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844745747 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844745763 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844745763Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844745778 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844745793 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844745793Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844745804 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844745818 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844745819 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@518eff78: reopening flushed file at 1733844745848 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6113182: reopening flushed file at 1733844745855 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65fbb519: reopening flushed file at 1733844745863 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@769ac750: reopening flushed file at 1733844745870 (+7 ms)Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 159ms, sequenceid=60, compaction requested=false at 1733844745878 (+8 ms)Writing region close event to WAL at 1733844745880 (+2 ms)Closed at 1733844745881 (+1 ms)
2024-12-10T15:32:25,881 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:25,881 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:25,881 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:25,881 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:25,882 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:25,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33755 is added to blk_1073741891_1074 (size=1045)
2024-12-10T15:32:25,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35247 is added to blk_1073741891_1074 (size=1045)
2024-12-10T15:32:26,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:26,614 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:27,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:27,489 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:32:27,509 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,509 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,510 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,510 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,511 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:27,614 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:28,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:28,615 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:29,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:29,616 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:30,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:30,616 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:30,882 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-10T15:32:30,882 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:32:30,882 INFO  [M:0;d6bcd7308055:44557 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:32:30,883 INFO  [M:0;d6bcd7308055:44557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44557
2024-12-10T15:32:30,883 INFO  [M:0;d6bcd7308055:44557 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:32:31,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:31,336 INFO  [M:0;d6bcd7308055:44557 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:32:31,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44557-0x100105a80090000, quorum=127.0.0.1:56326, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:32:31,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3948d06{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:31,339 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44c1c692{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:31,339 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:31,339 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5daabe6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:31,339 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1712bc01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:31,341 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:31,341 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 48d75f3e-a9c8-40d0-a309-761ae103fdcf) service to localhost/127.0.0.1:34973
2024-12-10T15:32:31,341 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data3/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:31,342 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data4/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:31,342 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:31,344 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:31,345 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:31,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c316933{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:31,353 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3fcb7a1e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:31,353 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:31,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a47bf3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:31,354 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4730afc2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:31,358 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:31,358 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:31,360 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:31,361 WARN  [BP-1301583617-172.17.0.2-1733844693229 heartbeating to localhost/127.0.0.1:34973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1301583617-172.17.0.2-1733844693229 (Datanode Uuid 0273a4e0-17eb-4e56-b28a-60eddd519f3e) service to localhost/127.0.0.1:34973
2024-12-10T15:32:31,362 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data5/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:31,362 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/cluster_76264465-b792-db5c-fb8e-4c054aea4bab/data/data6/current/BP-1301583617-172.17.0.2-1733844693229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:31,362 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:31,371 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@148f816d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:32:31,372 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59a85f94{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:31,372 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:31,372 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56d3681c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:31,372 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f149dad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:31,382 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:32:31,426 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:32:31,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:31,439 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=163 (was 81)
Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff0acbf55b8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34973
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:34973 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-15-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:34973 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:34973
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:34973 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34973
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34973
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34973
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35539
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35539
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-10
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff0acbf55b8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34973
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Command processor
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-11
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff0acbf55b8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-9
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34973
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=431 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 162) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4668 (was 4957)
2024-12-10T15:32:31,458 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=163, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=4654
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.log.dir so I do NOT create it in target/test-data/110e4321-7066-2018-f479-dc43703e9d46
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f67fdc8-d497-bce7-fa90-8ff2ab2f7c69/hadoop.tmp.dir so I do NOT create it in target/test-data/110e4321-7066-2018-f479-dc43703e9d46
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870, deleteOnExit=true
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/test.cache.data in system properties and HBase conf
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:32:31,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:32:31,459 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:32:31,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:32:31,461 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:32:31,461 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:32:31,461 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:32:31,461 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:32:31,476 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:32:31,617 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:31,949 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:31,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:31,955 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:31,965 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:31,965 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:31,965 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:32:31,970 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:31,972 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32305045{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:31,972 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@311e6ca3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:31,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T15:32:32,084 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fcc3202{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-34241-hadoop-hdfs-3_4_1-tests_jar-_-any-6038186850216224270/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:32:32,084 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34e28819{HTTP/1.1, (http/1.1)}{localhost:34241}
2024-12-10T15:32:32,084 INFO  [Time-limited test {}] server.Server(415): Started @166675ms
2024-12-10T15:32:32,098 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:32:32,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:32,566 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:32,570 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:32,573 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:32,573 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:32,573 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:32:32,574 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69604d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:32,574 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b49ea2d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:32,618 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:32,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6bd68f2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-38189-hadoop-hdfs-3_4_1-tests_jar-_-any-7207714833989043793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:32,707 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a6df939{HTTP/1.1, (http/1.1)}{localhost:38189}
2024-12-10T15:32:32,707 INFO  [Time-limited test {}] server.Server(415): Started @167298ms
2024-12-10T15:32:32,708 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:32,738 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:32,741 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:32,742 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:32,742 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:32,742 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:32:32,743 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11b04f0b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:32,743 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a857970{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:32,840 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e43587e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-39367-hadoop-hdfs-3_4_1-tests_jar-_-any-8079693882190897393/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:32,840 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4a49449a{HTTP/1.1, (http/1.1)}{localhost:39367}
2024-12-10T15:32:32,840 INFO  [Time-limited test {}] server.Server(415): Started @167431ms
2024-12-10T15:32:32,842 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:32,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:33,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:33,618 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:33,941 WARN  [Thread-1212 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data2/current/BP-110745400-172.17.0.2-1733844751490/current, will proceed with Du for space computation calculation, 
2024-12-10T15:32:33,941 WARN  [Thread-1211 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data1/current/BP-110745400-172.17.0.2-1733844751490/current, will proceed with Du for space computation calculation, 
2024-12-10T15:32:33,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:33,966 WARN  [Thread-1175 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:33,968 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6a71873179ff0c8 with lease ID 0x617d3bd6f0857e65: Processing first storage report for DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55 from datanode DatanodeRegistration(127.0.0.1:46495, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=39935, infoSecurePort=0, ipcPort=34289, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490)
2024-12-10T15:32:33,969 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6a71873179ff0c8 with lease ID 0x617d3bd6f0857e65: from storage DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55 node DatanodeRegistration(127.0.0.1:46495, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=39935, infoSecurePort=0, ipcPort=34289, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:33,969 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6a71873179ff0c8 with lease ID 0x617d3bd6f0857e65: Processing first storage report for DS-fe7d04ea-a3f0-4edb-a788-6672448057fe from datanode DatanodeRegistration(127.0.0.1:46495, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=39935, infoSecurePort=0, ipcPort=34289, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490)
2024-12-10T15:32:33,969 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6a71873179ff0c8 with lease ID 0x617d3bd6f0857e65: from storage DS-fe7d04ea-a3f0-4edb-a788-6672448057fe node DatanodeRegistration(127.0.0.1:46495, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=39935, infoSecurePort=0, ipcPort=34289, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:34,041 WARN  [Thread-1222 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data3/current/BP-110745400-172.17.0.2-1733844751490/current, will proceed with Du for space computation calculation, 
2024-12-10T15:32:34,041 WARN  [Thread-1223 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data4/current/BP-110745400-172.17.0.2-1733844751490/current, will proceed with Du for space computation calculation, 
2024-12-10T15:32:34,061 WARN  [Thread-1198 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:34,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8d16d3f2d29fc33b with lease ID 0x617d3bd6f0857e66: Processing first storage report for DS-44c46c64-0efa-4051-b4e0-59eabff78785 from datanode DatanodeRegistration(127.0.0.1:35651, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45135, infoSecurePort=0, ipcPort=45455, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490)
2024-12-10T15:32:34,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d16d3f2d29fc33b with lease ID 0x617d3bd6f0857e66: from storage DS-44c46c64-0efa-4051-b4e0-59eabff78785 node DatanodeRegistration(127.0.0.1:35651, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45135, infoSecurePort=0, ipcPort=45455, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:34,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8d16d3f2d29fc33b with lease ID 0x617d3bd6f0857e66: Processing first storage report for DS-0a252466-6f0f-476f-b078-ad801cc6ebcc from datanode DatanodeRegistration(127.0.0.1:35651, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45135, infoSecurePort=0, ipcPort=45455, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490)
2024-12-10T15:32:34,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d16d3f2d29fc33b with lease ID 0x617d3bd6f0857e66: from storage DS-0a252466-6f0f-476f-b078-ad801cc6ebcc node DatanodeRegistration(127.0.0.1:35651, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45135, infoSecurePort=0, ipcPort=45455, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:34,080 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46
2024-12-10T15:32:34,083 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/zookeeper_0, clientPort=54660, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:32:34,084 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54660
2024-12-10T15:32:34,085 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,086 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:32:34,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:32:34,100 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57 with version=8
2024-12-10T15:32:34,100 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:32:34,102 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:32:34,102 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,102 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:32:34,103 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43107
2024-12-10T15:32:34,105 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43107 connecting to ZooKeeper ensemble=127.0.0.1:54660
2024-12-10T15:32:34,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431070x0, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:32:34,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43107-0x100105b65690000 connected
2024-12-10T15:32:34,276 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,277 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,279 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:32:34,279 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57, hbase.cluster.distributed=false
2024-12-10T15:32:34,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:32:34,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43107
2024-12-10T15:32:34,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43107
2024-12-10T15:32:34,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43107
2024-12-10T15:32:34,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43107
2024-12-10T15:32:34,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43107
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:32:34,300 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:32:34,301 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:32:34,301 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41581
2024-12-10T15:32:34,302 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41581 connecting to ZooKeeper ensemble=127.0.0.1:54660
2024-12-10T15:32:34,303 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,305 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415810x0, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:32:34,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41581-0x100105b65690001 connected
2024-12-10T15:32:34,324 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:32:34,324 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:32:34,325 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:32:34,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:32:34,326 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:32:34,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41581
2024-12-10T15:32:34,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41581
2024-12-10T15:32:34,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41581
2024-12-10T15:32:34,334 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41581
2024-12-10T15:32:34,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41581
2024-12-10T15:32:34,347 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:43107
2024-12-10T15:32:34,348 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:32:34,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:32:34,362 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:32:34,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,375 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:32:34,375 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,43107,1733844754102 from backup master directory
2024-12-10T15:32:34,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:32:34,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,385 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:32:34,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:32:34,385 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,390 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/hbase.id] with ID: 153f4735-df08-4a9f-803e-3c03c41a064c
2024-12-10T15:32:34,390 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/.tmp/hbase.id
2024-12-10T15:32:34,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:32:34,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:32:34,398 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/.tmp/hbase.id]:[hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/hbase.id]
2024-12-10T15:32:34,422 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:34,422 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:32:34,423 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T15:32:34,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:34,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:32:34,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:32:34,456 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:32:34,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:32:34,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:32:34,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:32:34,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:32:34,484 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store
2024-12-10T15:32:34,491 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:32:34,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:32:34,492 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:34,492 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:32:34,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:34,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:34,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:32:34,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:34,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:32:34,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844754492Disabling compacts and flushes for region at 1733844754492Disabling writes for close at 1733844754493 (+1 ms)Writing region close event to WAL at 1733844754493Closed at 1733844754493
2024-12-10T15:32:34,494 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/.initializing
2024-12-10T15:32:34,494 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,497 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C43107%2C1733844754102, suffix=, logDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102, archiveDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/oldWALs, maxLogs=10
2024-12-10T15:32:34,497 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C43107%2C1733844754102.1733844754497
2024-12-10T15:32:34,503 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497
2024-12-10T15:32:34,504 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39935:39935),(127.0.0.1/127.0.0.1:45135:45135)]
2024-12-10T15:32:34,513 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:32:34,513 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:34,513 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,513 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,514 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:32:34,516 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:32:34,516 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,517 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:34,517 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:32:34,518 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:32:34,518 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,519 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:32:34,519 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:32:34,521 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:32:34,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,521 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:32:34,522 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:32:34,523 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:32:34,523 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,524 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:32:34,525 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,525 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,526 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,527 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,527 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,528 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:32:34,529 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:32:34,531 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:32:34,532 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=838238, jitterRate=0.0658755898475647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:32:34,533 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844754513Initializing all the Stores at 1733844754514 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844754514Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844754514Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844754514Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844754514Cleaning up temporary data from old regions at 1733844754527 (+13 ms)Region opened successfully at 1733844754533 (+6 ms)
2024-12-10T15:32:34,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:32:34,537 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26222681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:32:34,538 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:32:34,538 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:32:34,538 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:32:34,539 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:32:34,539 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:32:34,540 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:32:34,540 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:32:34,542 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:32:34,543 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:32:34,553 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:32:34,554 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:32:34,554 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:32:34,564 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:32:34,564 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:32:34,566 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:32:34,574 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:32:34,575 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:32:34,585 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:32:34,587 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:32:34,595 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:32:34,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:32:34,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:32:34,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,606 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,43107,1733844754102, sessionid=0x100105b65690000, setting cluster-up flag (Was=false)
2024-12-10T15:32:34,619 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:34,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,658 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:32:34,660 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:34,711 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:32:34,712 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,43107,1733844754102
2024-12-10T15:32:34,713 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:32:34,715 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:32:34,715 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:32:34,716 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:32:34,716 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,43107,1733844754102
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:32:34,717 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,718 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844784718
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:32:34,719 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,719 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:32:34,719 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:32:34,720 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:32:34,720 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:32:34,720 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:32:34,720 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:32:34,720 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:32:34,720 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,721 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:32:34,721 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844754720,5,FailOnTimeoutGroup]
2024-12-10T15:32:34,721 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844754721,5,FailOnTimeoutGroup]
2024-12-10T15:32:34,722 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,722 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:32:34,722 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,722 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:32:34,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:32:34,728 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:32:34,728 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57
2024-12-10T15:32:34,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:32:34,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:34,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:32:34,737 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:32:34,739 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:32:34,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,739 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:34,740 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:32:34,741 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:32:34,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,742 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(746): ClusterId : 153f4735-df08-4a9f-803e-3c03c41a064c
2024-12-10T15:32:34,742 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:32:34,743 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:34,743 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:32:34,745 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:32:34,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,745 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:34,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:32:34,747 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:32:34,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:34,747 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:34,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:32:34,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740
2024-12-10T15:32:34,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740
2024-12-10T15:32:34,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:32:34,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:32:34,750 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:32:34,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:32:34,754 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:32:34,754 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:32:34,754 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:32:34,754 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=783651, jitterRate=-0.003536418080329895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:32:34,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844754736Initializing all the Stores at 1733844754737 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844754737Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844754737Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844754737Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844754737Cleaning up temporary data from old regions at 1733844754750 (+13 ms)Region opened successfully at 1733844754755 (+5 ms)
2024-12-10T15:32:34,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:32:34,755 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:32:34,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:32:34,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:32:34,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:32:34,756 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:32:34,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844754755Disabling compacts and flushes for region at 1733844754755Disabling writes for close at 1733844754755Writing region close event to WAL at 1733844754756 (+1 ms)Closed at 1733844754756
2024-12-10T15:32:34,757 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:32:34,757 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:32:34,757 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:32:34,758 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:32:34,759 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:32:34,765 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:32:34,765 DEBUG [RS:0;d6bcd7308055:41581 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b49dcf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:32:34,776 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:41581
2024-12-10T15:32:34,776 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:32:34,776 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:32:34,776 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:32:34,777 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,43107,1733844754102 with port=41581, startcode=1733844754300
2024-12-10T15:32:34,778 DEBUG [RS:0;d6bcd7308055:41581 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:32:34,779 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44829, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:32:34,780 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43107 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,780 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43107 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,782 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57
2024-12-10T15:32:34,782 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40001
2024-12-10T15:32:34,782 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:32:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:32:34,792 DEBUG [RS:0;d6bcd7308055:41581 {}] zookeeper.ZKUtil(111): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,792 WARN  [RS:0;d6bcd7308055:41581 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:32:34,792 INFO  [RS:0;d6bcd7308055:41581 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:32:34,792 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,792 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,41581,1733844754300]
2024-12-10T15:32:34,796 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:32:34,798 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:32:34,798 INFO  [RS:0;d6bcd7308055:41581 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:32:34,798 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,798 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:32:34,799 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:32:34,799 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,799 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,799 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,799 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,799 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:32:34,800 DEBUG [RS:0;d6bcd7308055:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,802 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,41581,1733844754300-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:32:34,819 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:32:34,819 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,41581,1733844754300-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,819 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,819 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.Replication(171): d6bcd7308055,41581,1733844754300 started
2024-12-10T15:32:34,836 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:34,836 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,41581,1733844754300, RpcServer on d6bcd7308055/172.17.0.2:41581, sessionid=0x100105b65690001
2024-12-10T15:32:34,836 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:32:34,836 DEBUG [RS:0;d6bcd7308055:41581 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,836 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,41581,1733844754300'
2024-12-10T15:32:34,836 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:32:34,836 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,41581,1733844754300
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,41581,1733844754300'
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:32:34,837 DEBUG [RS:0;d6bcd7308055:41581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:32:34,837 INFO  [RS:0;d6bcd7308055:41581 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:32:34,838 INFO  [RS:0;d6bcd7308055:41581 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:32:34,909 WARN  [d6bcd7308055:43107 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:32:34,940 INFO  [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C41581%2C1733844754300, suffix=, logDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300, archiveDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs, maxLogs=32
2024-12-10T15:32:34,940 INFO  [RS:0;d6bcd7308055:41581 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:34,946 INFO  [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:34,946 DEBUG [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39935:39935),(127.0.0.1/127.0.0.1:45135:45135)]
2024-12-10T15:32:34,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:35,160 DEBUG [d6bcd7308055:43107 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:32:35,160 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:35,161 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,41581,1733844754300, state=OPENING
2024-12-10T15:32:35,244 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:32:35,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:35,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:32:35,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:32:35,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:32:35,255 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:32:35,255 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,41581,1733844754300}]
2024-12-10T15:32:35,408 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:32:35,410 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43541, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:32:35,413 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:32:35,414 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:32:35,415 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C41581%2C1733844754300.meta, suffix=.meta, logDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300, archiveDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs, maxLogs=32
2024-12-10T15:32:35,416 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta
2024-12-10T15:32:35,422 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta
2024-12-10T15:32:35,422 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45135:45135),(127.0.0.1/127.0.0.1:39935:39935)]
2024-12-10T15:32:35,423 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:32:35,423 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:32:35,424 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:32:35,424 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:32:35,424 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:32:35,424 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:35,424 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:32:35,424 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:32:35,425 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:32:35,426 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:32:35,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,427 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:35,427 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:32:35,428 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:32:35,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,429 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:35,429 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:32:35,430 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:32:35,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,430 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:35,430 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:32:35,431 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:32:35,431 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,432 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:32:35,432 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:32:35,433 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740
2024-12-10T15:32:35,434 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740
2024-12-10T15:32:35,436 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:32:35,436 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:32:35,436 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:32:35,438 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:32:35,439 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850890, jitterRate=0.08196280896663666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:32:35,439 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:32:35,440 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844755424Writing region info on filesystem at 1733844755424Initializing all the Stores at 1733844755425 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844755425Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844755425Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844755425Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844755425Cleaning up temporary data from old regions at 1733844755436 (+11 ms)Running coprocessor post-open hooks at 1733844755439 (+3 ms)Region opened successfully at 1733844755440 (+1 ms)
2024-12-10T15:32:35,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:35,441 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844755408
2024-12-10T15:32:35,444 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:32:35,445 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:32:35,445 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:35,446 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,41581,1733844754300, state=OPEN
2024-12-10T15:32:35,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:32:35,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:32:35,488 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:35,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:32:35,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:32:35,492 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:32:35,492 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,41581,1733844754300 in 233 msec
2024-12-10T15:32:35,495 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:32:35,495 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec
2024-12-10T15:32:35,496 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:32:35,496 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:32:35,498 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:32:35,499 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,41581,1733844754300, seqNum=-1]
2024-12-10T15:32:35,499 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:32:35,500 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:32:35,507 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 791 msec
2024-12-10T15:32:35,507 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844755507, completionTime=-1
2024-12-10T15:32:35,507 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:32:35,507 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:32:35,509 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844815510
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844875510
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:43107, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,511 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,513 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.130sec
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:32:35,515 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:32:35,518 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:32:35,518 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:32:35,518 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,43107,1733844754102-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:32:35,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37acc152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:32:35,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,43107,-1 for getting cluster id
2024-12-10T15:32:35,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:32:35,545 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '153f4735-df08-4a9f-803e-3c03c41a064c'
2024-12-10T15:32:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:32:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "153f4735-df08-4a9f-803e-3c03c41a064c"

2024-12-10T15:32:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc9235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:32:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,43107,-1]
2024-12-10T15:32:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:32:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:32:35,548 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:32:35,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f03c169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:32:35,550 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:32:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,41581,1733844754300, seqNum=-1]
2024-12-10T15:32:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:32:35,553 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:32:35,555 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,43107,1733844754102
2024-12-10T15:32:35,556 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:32:35,558 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:32:35,558 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-10T15:32:35,558 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-10T15:32:35,559 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T15:32:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is d6bcd7308055,43107,1733844754102
2024-12-10T15:32:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28855b6d
2024-12-10T15:32:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T15:32:35,562 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T15:32:35,562 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T15:32:35,562 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T15:32:35,562 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:32:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T15:32:35,566 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T15:32:35,566 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,566 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-10T15:32:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:32:35,567 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T15:32:35,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741835_1011 (size=395)
2024-12-10T15:32:35,575 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741835_1011 (size=395)
2024-12-10T15:32:35,577 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a2e0f9c6d07c61439f1af73be0c8e2b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57
2024-12-10T15:32:35,585 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35651 is added to blk_1073741836_1012 (size=78)
2024-12-10T15:32:35,585 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46495 is added to blk_1073741836_1012 (size=78)
2024-12-10T15:32:35,586 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:35,587 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 1a2e0f9c6d07c61439f1af73be0c8e2b, disabling compactions & flushes
2024-12-10T15:32:35,587 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,587 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,587 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b. after waiting 0 ms
2024-12-10T15:32:35,587 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,587 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,587 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a2e0f9c6d07c61439f1af73be0c8e2b:
Waiting for close lock at 1733844755587Disabling compacts and flushes for region at 1733844755587Disabling writes for close at 1733844755587Writing region close event to WAL at 1733844755587Closed at 1733844755587
2024-12-10T15:32:35,589 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T15:32:35,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733844755589"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844755589"}]},"ts":"1733844755589"}
2024-12-10T15:32:35,592 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T15:32:35,594 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T15:32:35,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844755594"}]},"ts":"1733844755594"}
2024-12-10T15:32:35,596 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-10T15:32:35,597 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1a2e0f9c6d07c61439f1af73be0c8e2b, ASSIGN}]
2024-12-10T15:32:35,598 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1a2e0f9c6d07c61439f1af73be0c8e2b, ASSIGN
2024-12-10T15:32:35,600 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1a2e0f9c6d07c61439f1af73be0c8e2b, ASSIGN; state=OFFLINE, location=d6bcd7308055,41581,1733844754300; forceNewPlan=false, retain=false
2024-12-10T15:32:35,620 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:35,750 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a2e0f9c6d07c61439f1af73be0c8e2b, regionState=OPENING, regionLocation=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:35,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1a2e0f9c6d07c61439f1af73be0c8e2b, ASSIGN because future has completed
2024-12-10T15:32:35,754 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a2e0f9c6d07c61439f1af73be0c8e2b, server=d6bcd7308055,41581,1733844754300}]
2024-12-10T15:32:35,912 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,912 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a2e0f9c6d07c61439f1af73be0c8e2b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:32:35,913 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,913 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:32:35,913 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,913 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,914 INFO  [StoreOpener-1a2e0f9c6d07c61439f1af73be0c8e2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1a2e0f9c6d07c61439f1af73be0c8e2b 
2024-12-10T15:32:35,916 INFO  [StoreOpener-1a2e0f9c6d07c61439f1af73be0c8e2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a2e0f9c6d07c61439f1af73be0c8e2b columnFamilyName info
2024-12-10T15:32:35,916 DEBUG [StoreOpener-1a2e0f9c6d07c61439f1af73be0c8e2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:32:35,916 INFO  [StoreOpener-1a2e0f9c6d07c61439f1af73be0c8e2b-1 {}] regionserver.HStore(327): Store=1a2e0f9c6d07c61439f1af73be0c8e2b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:32:35,917 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,917 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,918 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,918 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,918 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,920 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,922 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:32:35,923 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a2e0f9c6d07c61439f1af73be0c8e2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690177, jitterRate=-0.12239529192447662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:32:35,923 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:32:35,924 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a2e0f9c6d07c61439f1af73be0c8e2b:
Running coprocessor pre-open hook at 1733844755913Writing region info on filesystem at 1733844755913Initializing all the Stores at 1733844755914 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844755914Cleaning up temporary data from old regions at 1733844755918 (+4 ms)Running coprocessor post-open hooks at 1733844755923 (+5 ms)Region opened successfully at 1733844755924 (+1 ms)
2024-12-10T15:32:35,925 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b., pid=6, masterSystemTime=1733844755908
2024-12-10T15:32:35,928 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,928 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:35,929 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a2e0f9c6d07c61439f1af73be0c8e2b, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,41581,1733844754300
2024-12-10T15:32:35,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a2e0f9c6d07c61439f1af73be0c8e2b, server=d6bcd7308055,41581,1733844754300 because future has completed
2024-12-10T15:32:35,936 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T15:32:35,936 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a2e0f9c6d07c61439f1af73be0c8e2b, server=d6bcd7308055,41581,1733844754300 in 179 msec
2024-12-10T15:32:35,939 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T15:32:35,940 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1a2e0f9c6d07c61439f1af73be0c8e2b, ASSIGN in 339 msec
2024-12-10T15:32:35,941 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T15:32:35,941 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844755941"}]},"ts":"1733844755941"}
2024-12-10T15:32:35,943 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-10T15:32:35,945 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T15:32:35,947 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 382 msec
2024-12-10T15:32:35,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:36,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:36,620 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:36,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:37,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:37,489 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:32:37,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,507 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,507 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,508 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,511 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,511 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,512 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,514 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:32:37,621 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:37,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:38,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:38,622 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:38,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:39,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:39,622 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:39,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:40,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:40,623 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:40,796 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T15:32:40,797 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-10T15:32:40,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:41,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:41,623 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:41,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:41,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:32:41,986 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T15:32:41,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T15:32:41,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-10T15:32:41,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:32:41,988 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T15:32:41,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T15:32:41,988 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-10T15:32:42,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:42,624 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:42,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:43,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:43,625 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:43,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:44,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:44,625 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:44,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:45,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:32:45,620 INFO  [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-10T15:32:45,621 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-10T15:32:45,623 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T15:32:45,623 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:32:45,626 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b., hostname=d6bcd7308055,41581,1733844754300, seqNum=2]
2024-12-10T15:32:45,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:46,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:46,627 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:46,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:47,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:47,627 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:47,630 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:47,631 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,631 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,631 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,632 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK], DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]) is bad.
2024-12-10T15:32:47,632 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK], DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]) is bad.
2024-12-10T15:32:47,632 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK], DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35651,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]) is bad.
2024-12-10T15:32:47,632 WARN  [PacketResponder: BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35651] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,632 WARN  [PacketResponder: BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35651] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:43118 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35651:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43118 dst: /127.0.0.1:35651
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:40466 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40466 dst: /127.0.0.1:46495
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:43116 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35651:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43116 dst: /127.0.0.1:35651
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1907890231_22 at /127.0.0.1:40430 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40430 dst: /127.0.0.1:46495
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1907890231_22 at /127.0.0.1:43084 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35651:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43084 dst: /127.0.0.1:35651
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:40476 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40476 dst: /127.0.0.1:46495
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,711 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e43587e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:47,711 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4a49449a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:47,711 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:47,712 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a857970{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:47,712 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11b04f0b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:47,714 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:47,714 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:47,715 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid 8482eb9c-e0ac-498b-9252-d04f626bc870) service to localhost/127.0.0.1:40001
2024-12-10T15:32:47,715 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:47,715 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data3/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:47,715 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data4/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:47,716 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:47,724 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:47,727 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:47,733 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:47,733 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:47,733 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:32:47,733 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@301cb9c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:47,734 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@754bd44d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:47,836 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@551d415c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-42557-hadoop-hdfs-3_4_1-tests_jar-_-any-5448740702977181502/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:47,837 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a315286{HTTP/1.1, (http/1.1)}{localhost:42557}
2024-12-10T15:32:47,837 INFO  [Time-limited test {}] server.Server(415): Started @182428ms
2024-12-10T15:32:47,838 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:47,868 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,868 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,868 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:47,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:58420 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58420 dst: /127.0.0.1:46495
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:58410 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58410 dst: /127.0.0.1:46495
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1907890231_22 at /127.0.0.1:58398 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46495:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58398 dst: /127.0.0.1:46495
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:47,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6bd68f2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:47,870 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a6df939{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:47,870 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:47,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b49ea2d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:47,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69604d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:47,872 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:47,872 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:47,872 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0) service to localhost/127.0.0.1:40001
2024-12-10T15:32:47,872 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:47,872 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data1/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:47,872 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data2/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:47,873 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:47,881 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:47,884 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:47,885 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:47,885 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:47,885 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:32:47,885 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20e2c2f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:47,885 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ef9e5fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:47,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:47,984 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@438d77ef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-41093-hadoop-hdfs-3_4_1-tests_jar-_-any-1223405027917303525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:47,985 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3eb01649{HTTP/1.1, (http/1.1)}{localhost:41093}
2024-12-10T15:32:47,985 INFO  [Time-limited test {}] server.Server(415): Started @182576ms
2024-12-10T15:32:47,986 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:48,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:48,456 WARN  [Thread-1346 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:48,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9772cee41ea30647 with lease ID 0x617d3bd6f0857e67: from storage DS-44c46c64-0efa-4051-b4e0-59eabff78785 node DatanodeRegistration(127.0.0.1:44793, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45417, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:48,458 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9772cee41ea30647 with lease ID 0x617d3bd6f0857e67: from storage DS-0a252466-6f0f-476f-b078-ad801cc6ebcc node DatanodeRegistration(127.0.0.1:44793, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=45417, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:32:48,628 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:48,760 WARN  [Thread-1366 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:48,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x123b173ac829e0c with lease ID 0x617d3bd6f0857e68: from storage DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55 node DatanodeRegistration(127.0.0.1:37879, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=35981, infoSecurePort=0, ipcPort=34431, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:48,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x123b173ac829e0c with lease ID 0x617d3bd6f0857e68: from storage DS-fe7d04ea-a3f0-4edb-a788-6672448057fe node DatanodeRegistration(127.0.0.1:37879, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=35981, infoSecurePort=0, ipcPort=34431, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:48,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:49,006 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-10T15:32:49,009 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-10T15:32:49,011 ERROR [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:49,011 WARN  [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:49,011 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C41581%2C1733844754300:(num 1733844754940) roll requested
2024-12-10T15:32:49,012 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:49,018 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 newFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:49,018 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:49,019 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:49,019 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:49,019 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:49,019 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:49,019 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:49,020 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:49,020 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:49,020 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:49,020 WARN  [IPC Server handler 3 on default port 40001 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1015
2024-12-10T15:32:49,021 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 after 1ms
2024-12-10T15:32:49,021 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45417:45417),(127.0.0.1/127.0.0.1:35981:35981)]
2024-12-10T15:32:49,021 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 is not closed yet, will try archiving it next time
2024-12-10T15:32:49,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:49,629 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:49,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:50,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:50,629 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:50,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:51,024 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-10T15:32:51,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:51,458 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T15:32:51,630 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:51,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:52,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:52,631 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:52,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:53,021 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 after 4001ms
2024-12-10T15:32:53,028 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:37879,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:53,029 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44793,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK], DatanodeInfoWithStorage[127.0.0.1:37879,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37879,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]) is bad.
2024-12-10T15:32:53,029 WARN  [PacketResponder: BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37879] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:53,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:53444 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44793:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53444 dst: /127.0.0.1:44793
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:53,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:57906 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37879:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57906 dst: /127.0.0.1:37879
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:53,089 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@438d77ef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:53,090 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3eb01649{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:53,090 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:53,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ef9e5fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:53,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20e2c2f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:53,092 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:53,092 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:53,092 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:53,092 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0) service to localhost/127.0.0.1:40001
2024-12-10T15:32:53,093 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data1/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:53,093 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data2/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:53,093 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:53,103 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:53,106 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:53,106 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:53,106 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:53,106 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:32:53,107 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45e0e50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:53,107 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@239fd302{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:53,205 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f00b65f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-44683-hadoop-hdfs-3_4_1-tests_jar-_-any-8047953244300204111/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:53,206 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cff1d35{HTTP/1.1, (http/1.1)}{localhost:44683}
2024-12-10T15:32:53,206 INFO  [Time-limited test {}] server.Server(415): Started @187797ms
2024-12-10T15:32:53,207 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:53,239 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:53,240 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1033707050_22 at /127.0.0.1:43160 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44793:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43160 dst: /127.0.0.1:44793
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:53,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@551d415c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:53,246 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a315286{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:32:53,246 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:32:53,246 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@754bd44d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:32:53,246 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@301cb9c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:32:53,249 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:32:53,249 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:32:53,249 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:32:53,249 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid 8482eb9c-e0ac-498b-9252-d04f626bc870) service to localhost/127.0.0.1:40001
2024-12-10T15:32:53,250 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data3/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:53,250 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data4/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:32:53,250 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:32:53,259 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:32:53,262 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:32:53,262 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:32:53,262 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:32:53,262 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:32:53,263 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@319474d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:32:53,263 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71132282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:32:53,385 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420a3d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/java.io.tmpdir/jetty-localhost-38649-hadoop-hdfs-3_4_1-tests_jar-_-any-15238598432462229925/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:32:53,385 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e8ab63{HTTP/1.1, (http/1.1)}{localhost:38649}
2024-12-10T15:32:53,385 INFO  [Time-limited test {}] server.Server(415): Started @187976ms
2024-12-10T15:32:53,387 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:32:53,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:53,631 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:53,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:54,062 WARN  [Thread-1420 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:54,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87ee552b1dc8da46 with lease ID 0x617d3bd6f0857e69: from storage DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55 node DatanodeRegistration(127.0.0.1:46711, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=37535, infoSecurePort=0, ipcPort=36815, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:54,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87ee552b1dc8da46 with lease ID 0x617d3bd6f0857e69: from storage DS-fe7d04ea-a3f0-4edb-a788-6672448057fe node DatanodeRegistration(127.0.0.1:46711, datanodeUuid=dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0, infoPort=37535, infoSecurePort=0, ipcPort=36815, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:54,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:54,585 WARN  [Thread-1440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:32:54,587 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x19b4799ec0d3284d with lease ID 0x617d3bd6f0857e6a: from storage DS-44c46c64-0efa-4051-b4e0-59eabff78785 node DatanodeRegistration(127.0.0.1:34193, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=36407, infoSecurePort=0, ipcPort=43213, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:32:54,588 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x19b4799ec0d3284d with lease ID 0x617d3bd6f0857e6a: from storage DS-0a252466-6f0f-476f-b078-ad801cc6ebcc node DatanodeRegistration(127.0.0.1:34193, datanodeUuid=8482eb9c-e0ac-498b-9252-d04f626bc870, infoPort=36407, infoSecurePort=0, ipcPort=43213, storageInfo=lv=-57;cid=testClusterID;nsid=1691391361;c=1733844751490), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:32:54,616 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-10T15:32:54,618 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-10T15:32:54,619 ERROR [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44793,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:54,620 WARN  [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44793,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:54,620 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C41581%2C1733844754300:(num 1733844769012) roll requested
2024-12-10T15:32:54,620 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:54,625 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 newFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:54,625 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:54,625 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:54,625 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:54,625 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:54,626 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:54,626 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:54,626 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44793,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:54,626 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44793,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:54,626 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:54,627 WARN  [IPC Server handler 0 on default port 40001 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-10T15:32:54,627 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:36407:36407)]
2024-12-10T15:32:54,627 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 is not closed yet, will try archiving it next time
2024-12-10T15:32:54,627 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 after 1ms
2024-12-10T15:32:54,632 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:54,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:55,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:55,632 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:55,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:56,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:56,629 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:32:56,633 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:56,635 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620 newFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:32:56,635 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:56,635 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:56,636 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:56,636 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:56,636 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:32:56,636 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:32:56,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741838_1019 (size=1264)
2024-12-10T15:32:56,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741838_1019 (size=1264)
2024-12-10T15:32:56,639 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 is not closed yet, will try archiving it next time
2024-12-10T15:32:56,642 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:36407:36407)]
2024-12-10T15:32:56,642 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 is not closed yet, will try archiving it next time
2024-12-10T15:32:56,642 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:56,643 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:56,643 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940 after 0ms
2024-12-10T15:32:56,643 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:56,653 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733844755924/Put/vlen=218/seqid=0]
2024-12-10T15:32:56,653 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733844765629/Put/vlen=1045/seqid=0]
2024-12-10T15:32:56,653 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844754940
2024-12-10T15:32:56,654 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:56,654 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:56,654 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 after 0ms
2024-12-10T15:32:56,654 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:56,658 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733844769011/Put/vlen=1045/seqid=0]
2024-12-10T15:32:56,658 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733844771026/Put/vlen=1045/seqid=0]
2024-12-10T15:32:56,658 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012
2024-12-10T15:32:56,658 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:56,658 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:56,658 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620 after 0ms
2024-12-10T15:32:56,658 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844774620
2024-12-10T15:32:56,663 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733844774619/Put/vlen=1045/seqid=0]
2024-12-10T15:32:56,663 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:32:56,663 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:32:56,663 WARN  [IPC Server handler 2 on default port 40001 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-10T15:32:56,663 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 after 0ms
2024-12-10T15:32:56,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:57,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:57,594 WARN  [ResponseProcessor for block BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:57,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1907890231_22 at /127.0.0.1:56432 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:46711:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56432 dst: /127.0.0.1:46711
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46711 remote=/127.0.0.1:56432]. Total timeout mills is 60000, 59041 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:57,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1907890231_22 at /127.0.0.1:41788 [Receiving block BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34193:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41788 dst: /127.0.0.1:34193
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:32:57,594 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 block BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46711,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK], DatanodeInfoWithStorage[127.0.0.1:34193,DS-44c46c64-0efa-4051-b4e0-59eabff78785,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46711,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]) is bad.
2024-12-10T15:32:57,595 WARN  [DataStreamer for file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 block BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:32:57,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741839_1022 (size=85)
2024-12-10T15:32:57,634 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:57,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:58,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:58,628 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844769012 after 4002ms
2024-12-10T15:32:58,634 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:58,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:59,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:59,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:32:59,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:00,064 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T15:33:00,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:00,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:00,664 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 after 4001ms
2024-12-10T15:33:00,664 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:33:00,669 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:33:00,669 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-10T15:33:00,670 ERROR [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,670 WARN  [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,670 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C41581%2C1733844754300.meta:.meta(num 1733844755416) roll requested
2024-12-10T15:33:00,670 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.meta.1733844780670.meta
2024-12-10T15:33:00,676 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,676 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,677 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,677 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,677 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,677 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844780670.meta
2024-12-10T15:33:00,678 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,678 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,678 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta
2024-12-10T15:33:00,678 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36407:36407),(127.0.0.1/127.0.0.1:37535:37535)]
2024-12-10T15:33:00,678 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta is not closed yet, will try archiving it next time
2024-12-10T15:33:00,679 WARN  [IPC Server handler 0 on default port 40001 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014
2024-12-10T15:33:00,679 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta after 1ms
2024-12-10T15:33:00,697 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/info/cc9d034307034adcb6a28f8d23fc9ea1 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b./info:regioninfo/1733844755929/Put/seqid=0
2024-12-10T15:33:00,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741841_1025 (size=7125)
2024-12-10T15:33:00,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741841_1025 (size=7125)
2024-12-10T15:33:00,703 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/info/cc9d034307034adcb6a28f8d23fc9ea1
2024-12-10T15:33:00,722 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/ns/48485582861e47c89b0e1ddd959d5609 is 43, key is default/ns:d/1733844755501/Put/seqid=0
2024-12-10T15:33:00,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741842_1026 (size=5153)
2024-12-10T15:33:00,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741842_1026 (size=5153)
2024-12-10T15:33:00,728 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/ns/48485582861e47c89b0e1ddd959d5609
2024-12-10T15:33:00,750 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/table/8db9365aea824ade93a8ba105f16b2fb is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733844755941/Put/seqid=0
2024-12-10T15:33:00,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741843_1027 (size=5438)
2024-12-10T15:33:00,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741843_1027 (size=5438)
2024-12-10T15:33:00,755 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/table/8db9365aea824ade93a8ba105f16b2fb
2024-12-10T15:33:00,764 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/info/cc9d034307034adcb6a28f8d23fc9ea1 as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/info/cc9d034307034adcb6a28f8d23fc9ea1
2024-12-10T15:33:00,773 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/info/cc9d034307034adcb6a28f8d23fc9ea1, entries=10, sequenceid=11, filesize=7.0 K
2024-12-10T15:33:00,774 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/ns/48485582861e47c89b0e1ddd959d5609 as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/ns/48485582861e47c89b0e1ddd959d5609
2024-12-10T15:33:00,780 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/ns/48485582861e47c89b0e1ddd959d5609, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T15:33:00,781 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/.tmp/table/8db9365aea824ade93a8ba105f16b2fb as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/table/8db9365aea824ade93a8ba105f16b2fb
2024-12-10T15:33:00,787 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/table/8db9365aea824ade93a8ba105f16b2fb, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T15:33:00,789 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 119ms, sequenceid=11, compaction requested=false
2024-12-10T15:33:00,789 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T15:33:00,789 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a2e0f9c6d07c61439f1af73be0c8e2b 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-10T15:33:00,789 ERROR [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,789 WARN  [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57-prefix:d6bcd7308055,41581,1733844754300 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,790 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C41581%2C1733844754300:(num 1733844776629) roll requested
2024-12-10T15:33:00,790 INFO  [regionserver/d6bcd7308055:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C41581%2C1733844754300.1733844780790
2024-12-10T15:33:00,795 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 newFile=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844780790
2024-12-10T15:33:00,795 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,796 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,796 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,796 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,796 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:00,796 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844780790
2024-12-10T15:33:00,796 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,797 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36407:36407),(127.0.0.1/127.0.0.1:37535:37535)]
2024-12-10T15:33:00,797 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-110745400-172.17.0.2-1733844751490:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:00,797 DEBUG [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 is not closed yet, will try archiving it next time
2024-12-10T15:33:00,797 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:33:00,798 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 after 1ms
2024-12-10T15:33:00,799 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.1733844776629 to hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs/d6bcd7308055%2C41581%2C1733844754300.1733844776629
2024-12-10T15:33:00,815 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/.tmp/info/ec831da181904efc8c05931c337e2dbd is 1080, key is row1002/info:/1733844765629/Put/seqid=0
2024-12-10T15:33:00,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741845_1029 (size=9270)
2024-12-10T15:33:00,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741845_1029 (size=9270)
2024-12-10T15:33:00,821 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/.tmp/info/ec831da181904efc8c05931c337e2dbd
2024-12-10T15:33:00,829 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/.tmp/info/ec831da181904efc8c05931c337e2dbd as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/info/ec831da181904efc8c05931c337e2dbd
2024-12-10T15:33:00,835 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/info/ec831da181904efc8c05931c337e2dbd, entries=4, sequenceid=8, filesize=9.1 K
2024-12-10T15:33:00,836 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 1a2e0f9c6d07c61439f1af73be0c8e2b in 47ms, sequenceid=8, compaction requested=false
2024-12-10T15:33:00,836 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1a2e0f9c6d07c61439f1af73be0c8e2b:

2024-12-10T15:33:00,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:33:00,841 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:33:00,841 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:33:00,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:00,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:00,842 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:33:00,842 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:33:00,842 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=672875540, stopped=false
2024-12-10T15:33:00,842 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,43107,1733844754102
2024-12-10T15:33:00,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:33:00,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:33:00,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:00,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:00,958 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:33:00,958 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:33:00,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:33:00,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:00,959 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:33:00,959 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:33:00,959 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,41581,1733844754300' *****
2024-12-10T15:33:00,959 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:33:00,959 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:33:00,959 INFO  [RS:0;d6bcd7308055:41581 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:33:00,959 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,41581,1733844754300
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:41581.
2024-12-10T15:33:00,960 DEBUG [RS:0;d6bcd7308055:41581 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:33:00,960 DEBUG [RS:0;d6bcd7308055:41581 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:00,960 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a2e0f9c6d07c61439f1af73be0c8e2b, disabling compactions & flushes
2024-12-10T15:33:00,960 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:33:00,960 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:33:00,960 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b. after waiting 0 ms
2024-12-10T15:33:00,960 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:33:00,961 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:33:00,961 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T15:33:00,961 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1a2e0f9c6d07c61439f1af73be0c8e2b=TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.}
2024-12-10T15:33:00,961 DEBUG [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a2e0f9c6d07c61439f1af73be0c8e2b
2024-12-10T15:33:00,961 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:33:00,961 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:33:00,961 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:33:00,961 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:33:00,961 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:33:00,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:00,968 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T15:33:00,968 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/data/default/TestLogRolling-testLogRollOnPipelineRestart/1a2e0f9c6d07c61439f1af73be0c8e2b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-10T15:33:00,969 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:33:00,969 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:33:00,969 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:33:00,969 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a2e0f9c6d07c61439f1af73be0c8e2b:
Waiting for close lock at 1733844780960Running coprocessor pre-close hooks at 1733844780960Disabling compacts and flushes for region at 1733844780960Disabling writes for close at 1733844780960Writing region close event to WAL at 1733844780961 (+1 ms)Running coprocessor post-close hooks at 1733844780969 (+8 ms)Closed at 1733844780969
2024-12-10T15:33:00,969 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844780961Running coprocessor pre-close hooks at 1733844780961Disabling compacts and flushes for region at 1733844780961Disabling writes for close at 1733844780961Writing region close event to WAL at 1733844780962 (+1 ms)Running coprocessor post-close hooks at 1733844780969 (+7 ms)Closed at 1733844780969
2024-12-10T15:33:00,969 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733844755562.1a2e0f9c6d07c61439f1af73be0c8e2b.
2024-12-10T15:33:00,969 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:33:01,161 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,41581,1733844754300; all regions closed.
2024-12-10T15:33:01,162 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:01,162 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:01,162 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:01,162 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:01,162 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:01,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741840_1023 (size=825)
2024-12-10T15:33:01,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741840_1023 (size=825)
2024-12-10T15:33:01,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:01,636 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:01,885 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T15:33:01,885 INFO  [regionserver/d6bcd7308055:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T15:33:01,968 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:01,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:33:01,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:33:01,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T15:33:02,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:02,587 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T15:33:02,636 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:02,804 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:33:02,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:03,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:03,637 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:03,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:04,080 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:33:04,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:04,638 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:04,680 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta after 4002ms
2024-12-10T15:33:04,681 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/WALs/d6bcd7308055,41581,1733844754300/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta to hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs/d6bcd7308055%2C41581%2C1733844754300.meta.1733844755416.meta
2024-12-10T15:33:04,685 DEBUG [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs
2024-12-10T15:33:04,685 INFO  [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C41581%2C1733844754300.meta:.meta(num 1733844780670)
2024-12-10T15:33:04,685 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,686 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,686 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,686 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,686 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741844_1028 (size=1162)
2024-12-10T15:33:04,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741844_1028 (size=1162)
2024-12-10T15:33:04,695 DEBUG [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs
2024-12-10T15:33:04,695 INFO  [RS:0;d6bcd7308055:41581 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C41581%2C1733844754300:(num 1733844780790)
2024-12-10T15:33:04,695 DEBUG [RS:0;d6bcd7308055:41581 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:04,695 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:33:04,695 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:33:04,696 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:33:04,696 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:33:04,696 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:33:04,696 INFO  [RS:0;d6bcd7308055:41581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41581
2024-12-10T15:33:04,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:33:04,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,41581,1733844754300
2024-12-10T15:33:04,715 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:33:04,716 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,41581,1733844754300]
2024-12-10T15:33:04,737 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,41581,1733844754300 already deleted, retry=false
2024-12-10T15:33:04,737 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,41581,1733844754300 expired; onlineServers=0
2024-12-10T15:33:04,737 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,43107,1733844754102' *****
2024-12-10T15:33:04,737 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:33:04,737 INFO  [M:0;d6bcd7308055:43107 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:33:04,737 INFO  [M:0;d6bcd7308055:43107 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:33:04,737 DEBUG [M:0;d6bcd7308055:43107 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:33:04,737 DEBUG [M:0;d6bcd7308055:43107 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:33:04,737 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:33:04,737 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844754721 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844754721,5,FailOnTimeoutGroup]
2024-12-10T15:33:04,737 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844754720 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844754720,5,FailOnTimeoutGroup]
2024-12-10T15:33:04,737 INFO  [M:0;d6bcd7308055:43107 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:33:04,738 INFO  [M:0;d6bcd7308055:43107 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:33:04,738 DEBUG [M:0;d6bcd7308055:43107 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:33:04,738 INFO  [M:0;d6bcd7308055:43107 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:33:04,738 INFO  [M:0;d6bcd7308055:43107 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:33:04,738 INFO  [M:0;d6bcd7308055:43107 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:33:04,738 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:33:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:33:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:04,762 DEBUG [M:0;d6bcd7308055:43107 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-10T15:33:04,762 DEBUG [M:0;d6bcd7308055:43107 {}] master.ActiveMasterManager(353): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-10T15:33:04,763 INFO  [M:0;d6bcd7308055:43107 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/.lastflushedseqids
2024-12-10T15:33:04,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741846_1030 (size=111)
2024-12-10T15:33:04,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741846_1030 (size=111)
2024-12-10T15:33:04,772 INFO  [M:0;d6bcd7308055:43107 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:33:04,772 INFO  [M:0;d6bcd7308055:43107 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:33:04,772 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:33:04,772 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:04,772 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:04,772 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:33:04,773 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:04,773 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-12-10T15:33:04,773 ERROR [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData-prefix:d6bcd7308055,43107,1733844754102 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:04,773 WARN  [FSHLog-0-hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData-prefix:d6bcd7308055,43107,1733844754102 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:04,773 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d6bcd7308055%2C43107%2C1733844754102:(num 1733844754497) roll requested
2024-12-10T15:33:04,774 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C43107%2C1733844754102.1733844784773
2024-12-10T15:33:04,779 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,779 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,779 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,779 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,779 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:04,780 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844784773
2024-12-10T15:33:04,780 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:04,780 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46495,DS-ca6835e2-abe2-45c3-9058-e8bb48f48f55,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T15:33:04,780 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497
2024-12-10T15:33:04,781 WARN  [IPC Server handler 0 on default port 40001 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-12-10T15:33:04,781 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 after 1ms
2024-12-10T15:33:04,785 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:36407:36407)]
2024-12-10T15:33:04,785 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 is not closed yet, will try archiving it next time
2024-12-10T15:33:04,801 DEBUG [M:0;d6bcd7308055:43107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5d9c6b424014c269f9782d4fe1e6aaf is 82, key is hbase:meta,,1/info:regioninfo/1733844755444/Put/seqid=0
2024-12-10T15:33:04,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741848_1033 (size=5672)
2024-12-10T15:33:04,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741848_1033 (size=5672)
2024-12-10T15:33:04,813 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5d9c6b424014c269f9782d4fe1e6aaf
2024-12-10T15:33:04,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:33:04,826 INFO  [RS:0;d6bcd7308055:41581 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:33:04,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x100105b65690001, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:33:04,826 INFO  [RS:0;d6bcd7308055:41581 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,41581,1733844754300; zookeeper connection closed.
2024-12-10T15:33:04,826 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1129ff57 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1129ff57
2024-12-10T15:33:04,827 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:33:04,832 DEBUG [M:0;d6bcd7308055:43107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc490a7998e74e95ac059f58c36c9658 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733844755946/Put/seqid=0
2024-12-10T15:33:04,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741849_1034 (size=6118)
2024-12-10T15:33:04,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741849_1034 (size=6118)
2024-12-10T15:33:04,839 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc490a7998e74e95ac059f58c36c9658
2024-12-10T15:33:04,857 DEBUG [M:0;d6bcd7308055:43107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bed7369f2aae4f36830d71d8f268c40e is 69, key is d6bcd7308055,41581,1733844754300/rs:state/1733844754780/Put/seqid=0
2024-12-10T15:33:04,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741850_1035 (size=5156)
2024-12-10T15:33:04,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741850_1035 (size=5156)
2024-12-10T15:33:04,862 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bed7369f2aae4f36830d71d8f268c40e
2024-12-10T15:33:04,884 DEBUG [M:0;d6bcd7308055:43107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9dcadee477dc44b4a29ba61e465d06db is 52, key is load_balancer_on/state:d/1733844755557/Put/seqid=0
2024-12-10T15:33:04,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741851_1036 (size=5056)
2024-12-10T15:33:04,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741851_1036 (size=5056)
2024-12-10T15:33:04,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:05,289 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9dcadee477dc44b4a29ba61e465d06db
2024-12-10T15:33:05,295 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b5d9c6b424014c269f9782d4fe1e6aaf as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b5d9c6b424014c269f9782d4fe1e6aaf
2024-12-10T15:33:05,302 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b5d9c6b424014c269f9782d4fe1e6aaf, entries=8, sequenceid=56, filesize=5.5 K
2024-12-10T15:33:05,303 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc490a7998e74e95ac059f58c36c9658 as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc490a7998e74e95ac059f58c36c9658
2024-12-10T15:33:05,308 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc490a7998e74e95ac059f58c36c9658, entries=6, sequenceid=56, filesize=6.0 K
2024-12-10T15:33:05,309 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bed7369f2aae4f36830d71d8f268c40e as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bed7369f2aae4f36830d71d8f268c40e
2024-12-10T15:33:05,314 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bed7369f2aae4f36830d71d8f268c40e, entries=1, sequenceid=56, filesize=5.0 K
2024-12-10T15:33:05,315 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9dcadee477dc44b4a29ba61e465d06db as hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9dcadee477dc44b4a29ba61e465d06db
2024-12-10T15:33:05,321 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9dcadee477dc44b4a29ba61e465d06db, entries=1, sequenceid=56, filesize=4.9 K
2024-12-10T15:33:05,322 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 549ms, sequenceid=56, compaction requested=false
2024-12-10T15:33:05,323 INFO  [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:05,323 DEBUG [M:0;d6bcd7308055:43107 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844784772Disabling compacts and flushes for region at 1733844784772Disabling writes for close at 1733844784772Obtaining lock to block concurrent updates at 1733844784773 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844784773Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733844784773Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844784785 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844784785Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844784800 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844784801 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844784818 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844784832 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844784832Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844784844 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844784857 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844784857Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844784866 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844784883 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844784883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@695f3b83: reopening flushed file at 1733844785294 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1d0df3: reopening flushed file at 1733844785302 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b074db2: reopening flushed file at 1733844785308 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468d7954: reopening flushed file at 1733844785314 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 549ms, sequenceid=56, compaction requested=false at 1733844785322 (+8 ms)Writing region close event to WAL at 1733844785323 (+1 ms)Closed at 1733844785323
2024-12-10T15:33:05,323 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:05,323 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:05,324 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:05,324 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:05,324 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:05,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34193 is added to blk_1073741847_1031 (size=757)
2024-12-10T15:33:05,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46711 is added to blk_1073741847_1031 (size=757)
2024-12-10T15:33:05,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:05,587 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T15:33:05,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:05,969 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,970 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:05,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:06,497 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:33:06,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:06,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:06,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:07,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:07,640 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:07,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:08,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:08,640 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:08,782 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 after 4001ms
2024-12-10T15:33:08,782 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/WALs/d6bcd7308055,43107,1733844754102/d6bcd7308055%2C43107%2C1733844754102.1733844754497 to hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/oldWALs/d6bcd7308055%2C43107%2C1733844754102.1733844754497
2024-12-10T15:33:08,785 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/MasterData/oldWALs/d6bcd7308055%2C43107%2C1733844754102.1733844754497 to hdfs://localhost:40001/user/jenkins/test-data/bff8506e-788e-2ce9-470b-59edc7dc1c57/oldWALs/d6bcd7308055%2C43107%2C1733844754102.1733844754497$masterlocalwal$
2024-12-10T15:33:08,786 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:33:08,786 INFO  [M:0;d6bcd7308055:43107 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:33:08,786 INFO  [M:0;d6bcd7308055:43107 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43107
2024-12-10T15:33:08,787 INFO  [M:0;d6bcd7308055:43107 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:33:08,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:33:08,952 INFO  [M:0;d6bcd7308055:43107 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:33:08,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43107-0x100105b65690000, quorum=127.0.0.1:54660, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:33:08,955 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420a3d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:33:08,955 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e8ab63{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:33:08,955 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:33:08,955 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71132282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:33:08,955 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@319474d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:33:08,957 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:33:08,957 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:33:08,957 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:33:08,957 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid 8482eb9c-e0ac-498b-9252-d04f626bc870) service to localhost/127.0.0.1:40001
2024-12-10T15:33:08,957 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data3/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:33:08,957 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data4/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:33:08,958 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:33:08,962 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f00b65f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:33:08,963 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cff1d35{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:33:08,963 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:33:08,963 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@239fd302{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:33:08,963 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45e0e50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:33:08,964 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:33:08,964 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:33:08,964 WARN  [BP-110745400-172.17.0.2-1733844751490 heartbeating to localhost/127.0.0.1:40001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-110745400-172.17.0.2-1733844751490 (Datanode Uuid dbb087c5-a9be-4cd1-9c18-5efa16c8fdc0) service to localhost/127.0.0.1:40001
2024-12-10T15:33:08,964 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:33:08,965 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data1/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:33:08,965 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/cluster_4a8a8d7c-1a27-6b90-4bdb-ae221862b870/data/data2/current/BP-110745400-172.17.0.2-1733844751490 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:33:08,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:09,022 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:33:09,030 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fcc3202{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:33:09,030 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34e28819{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:33:09,031 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:33:09,031 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@311e6ca3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:33:09,031 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32305045{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir/,STOPPED}
2024-12-10T15:33:09,037 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:33:09,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:33:09,077 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=183 (was 163)
Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40001
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40001 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40001 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40001
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40001 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40001
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:40001
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40001
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 431) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 261) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6476 (was 4654) - AvailableMemoryMB LEAK? -
2024-12-10T15:33:09,084 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=183, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=6476
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.log.dir so I do NOT create it in target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/110e4321-7066-2018-f479-dc43703e9d46/hadoop.tmp.dir so I do NOT create it in target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090, deleteOnExit=true
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/test.cache.data in system properties and HBase conf
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:33:09,085 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:33:09,086 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:33:09,086 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:33:09,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:33:09,102 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:33:09,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:09,641 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:09,647 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:33:09,651 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:33:09,653 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:33:09,653 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:33:09,653 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:33:09,657 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:33:09,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c518279{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:33:09,658 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ac313e8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:33:09,753 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f209b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/java.io.tmpdir/jetty-localhost-41153-hadoop-hdfs-3_4_1-tests_jar-_-any-6141409676919625480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:33:09,754 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@489be359{HTTP/1.1, (http/1.1)}{localhost:41153}
2024-12-10T15:33:09,754 INFO  [Time-limited test {}] server.Server(415): Started @204344ms
2024-12-10T15:33:09,766 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:33:09,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:10,177 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:33:10,181 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:33:10,182 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:33:10,182 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:33:10,182 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:33:10,182 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bc13808{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:33:10,183 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9590158{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:33:10,277 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@795929af{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/java.io.tmpdir/jetty-localhost-45165-hadoop-hdfs-3_4_1-tests_jar-_-any-6679926007634793391/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:33:10,277 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@23e7edd{HTTP/1.1, (http/1.1)}{localhost:45165}
2024-12-10T15:33:10,277 INFO  [Time-limited test {}] server.Server(415): Started @204868ms
2024-12-10T15:33:10,279 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:33:10,302 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:33:10,305 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:33:10,306 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:33:10,306 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:33:10,306 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:33:10,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@453c7069{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:33:10,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@796d466{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:33:10,397 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ca8115c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/java.io.tmpdir/jetty-localhost-38351-hadoop-hdfs-3_4_1-tests_jar-_-any-11270467465699434433/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:33:10,398 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7fd5b514{HTTP/1.1, (http/1.1)}{localhost:38351}
2024-12-10T15:33:10,398 INFO  [Time-limited test {}] server.Server(415): Started @204989ms
2024-12-10T15:33:10,399 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:33:10,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:10,641 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:10,975 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:11,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:11,642 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:11,842 WARN  [Thread-1660 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data1/current/BP-668363613-172.17.0.2-1733844789113/current, will proceed with Du for space computation calculation, 
2024-12-10T15:33:11,842 WARN  [Thread-1661 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data2/current/BP-668363613-172.17.0.2-1733844789113/current, will proceed with Du for space computation calculation, 
2024-12-10T15:33:11,858 WARN  [Thread-1624 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:33:11,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ead7b8a69feb80 with lease ID 0x5c8905d5afdb56a5: Processing first storage report for DS-96b71c11-5529-4f8d-87e6-966c82fd6401 from datanode DatanodeRegistration(127.0.0.1:39869, datanodeUuid=e4710c35-46f9-468d-8223-7f18419cbc0c, infoPort=46275, infoSecurePort=0, ipcPort=41567, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113)
2024-12-10T15:33:11,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ead7b8a69feb80 with lease ID 0x5c8905d5afdb56a5: from storage DS-96b71c11-5529-4f8d-87e6-966c82fd6401 node DatanodeRegistration(127.0.0.1:39869, datanodeUuid=e4710c35-46f9-468d-8223-7f18419cbc0c, infoPort=46275, infoSecurePort=0, ipcPort=41567, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:33:11,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ead7b8a69feb80 with lease ID 0x5c8905d5afdb56a5: Processing first storage report for DS-5c3c1878-c138-4076-9d7f-edd25da25246 from datanode DatanodeRegistration(127.0.0.1:39869, datanodeUuid=e4710c35-46f9-468d-8223-7f18419cbc0c, infoPort=46275, infoSecurePort=0, ipcPort=41567, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113)
2024-12-10T15:33:11,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ead7b8a69feb80 with lease ID 0x5c8905d5afdb56a5: from storage DS-5c3c1878-c138-4076-9d7f-edd25da25246 node DatanodeRegistration(127.0.0.1:39869, datanodeUuid=e4710c35-46f9-468d-8223-7f18419cbc0c, infoPort=46275, infoSecurePort=0, ipcPort=41567, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:33:11,975 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:11,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T15:33:12,044 WARN  [Thread-1671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data3/current/BP-668363613-172.17.0.2-1733844789113/current, will proceed with Du for space computation calculation, 
2024-12-10T15:33:12,044 WARN  [Thread-1672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data4/current/BP-668363613-172.17.0.2-1733844789113/current, will proceed with Du for space computation calculation, 
2024-12-10T15:33:12,062 WARN  [Thread-1647 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:33:12,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb977cc6cfc820d1b with lease ID 0x5c8905d5afdb56a6: Processing first storage report for DS-58e303c3-15ad-47b0-bdc5-91b1919d84c7 from datanode DatanodeRegistration(127.0.0.1:40271, datanodeUuid=1302ef6d-1452-4d94-9aa5-f4bc642d3123, infoPort=43363, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113)
2024-12-10T15:33:12,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb977cc6cfc820d1b with lease ID 0x5c8905d5afdb56a6: from storage DS-58e303c3-15ad-47b0-bdc5-91b1919d84c7 node DatanodeRegistration(127.0.0.1:40271, datanodeUuid=1302ef6d-1452-4d94-9aa5-f4bc642d3123, infoPort=43363, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:33:12,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb977cc6cfc820d1b with lease ID 0x5c8905d5afdb56a6: Processing first storage report for DS-b08a154e-194b-4e9e-8a67-4cfc1bbcab16 from datanode DatanodeRegistration(127.0.0.1:40271, datanodeUuid=1302ef6d-1452-4d94-9aa5-f4bc642d3123, infoPort=43363, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113)
2024-12-10T15:33:12,064 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb977cc6cfc820d1b with lease ID 0x5c8905d5afdb56a6: from storage DS-b08a154e-194b-4e9e-8a67-4cfc1bbcab16 node DatanodeRegistration(127.0.0.1:40271, datanodeUuid=1302ef6d-1452-4d94-9aa5-f4bc642d3123, infoPort=43363, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=1973390165;c=1733844789113), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:33:12,131 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10
2024-12-10T15:33:12,158 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/zookeeper_0, clientPort=58015, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:33:12,159 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58015
2024-12-10T15:33:12,160 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,162 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:33:12,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:33:12,174 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d with version=8
2024-12-10T15:33:12,174 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:33:12,176 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:33:12,176 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,176 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:33:12,177 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34157
2024-12-10T15:33:12,179 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34157 connecting to ZooKeeper ensemble=127.0.0.1:58015
2024-12-10T15:33:12,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:341570x0, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:33:12,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34157-0x100105bfa230000 connected
2024-12-10T15:33:12,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:12,478 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,479 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,482 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:33:12,482 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d, hbase.cluster.distributed=false
2024-12-10T15:33:12,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:33:12,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34157
2024-12-10T15:33:12,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34157
2024-12-10T15:33:12,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34157
2024-12-10T15:33:12,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34157
2024-12-10T15:33:12,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34157
2024-12-10T15:33:12,503 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:33:12,504 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:33:12,505 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:33:12,506 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36083
2024-12-10T15:33:12,509 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36083 connecting to ZooKeeper ensemble=127.0.0.1:58015
2024-12-10T15:33:12,509 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,511 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360830x0, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:33:12,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360830x0, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:33:12,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36083-0x100105bfa230001 connected
2024-12-10T15:33:12,526 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:33:12,527 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:33:12,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:33:12,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:33:12,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36083
2024-12-10T15:33:12,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36083
2024-12-10T15:33:12,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36083
2024-12-10T15:33:12,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36083
2024-12-10T15:33:12,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36083
2024-12-10T15:33:12,545 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:34157
2024-12-10T15:33:12,545 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:33:12,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:33:12,557 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:33:12,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,568 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:33:12,568 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,34157,1733844792176 from backup master directory
2024-12-10T15:33:12,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:33:12,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:33:12,578 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:33:12,578 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,584 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/hbase.id] with ID: b6dbaf2f-b370-495a-8c25-de4af7bd0b71
2024-12-10T15:33:12,584 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/.tmp/hbase.id
2024-12-10T15:33:12,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:33:12,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:33:12,592 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/.tmp/hbase.id]:[hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/hbase.id]
2024-12-10T15:33:12,603 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:12,603 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:33:12,605 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T15:33:12,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:33:12,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:33:12,626 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:33:12,627 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:33:12,627 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:33:12,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:33:12,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:33:12,635 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store
2024-12-10T15:33:12,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:33:12,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:33:12,641 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:12,641 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:33:12,641 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844792641Disabling compacts and flushes for region at 1733844792641Disabling writes for close at 1733844792641Writing region close event to WAL at 1733844792641Closed at 1733844792641
2024-12-10T15:33:12,642 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/.initializing
2024-12-10T15:33:12,642 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/WALs/d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,643 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:12,644 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C34157%2C1733844792176, suffix=, logDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/WALs/d6bcd7308055,34157,1733844792176, archiveDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/oldWALs, maxLogs=10
2024-12-10T15:33:12,645 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C34157%2C1733844792176.1733844792645
2024-12-10T15:33:12,649 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/WALs/d6bcd7308055,34157,1733844792176/d6bcd7308055%2C34157%2C1733844792176.1733844792645
2024-12-10T15:33:12,657 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46275:46275),(127.0.0.1/127.0.0.1:43363:43363)]
2024-12-10T15:33:12,658 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:33:12,658 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:12,658 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,658 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,659 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:33:12,661 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:33:12,661 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,661 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:12,661 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:33:12,662 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:33:12,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,663 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:33:12,663 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:33:12,664 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:33:12,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,665 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:33:12,665 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:33:12,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:33:12,666 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:33:12,667 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,667 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,667 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,669 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,669 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,669 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:33:12,670 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:33:12,672 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:33:12,673 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=748807, jitterRate=-0.047843143343925476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:33:12,674 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844792658Initializing all the Stores at 1733844792659 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844792659Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844792659Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844792659Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844792659Cleaning up temporary data from old regions at 1733844792669 (+10 ms)Region opened successfully at 1733844792674 (+5 ms)
2024-12-10T15:33:12,677 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:33:12,680 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd2ff80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:33:12,681 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:33:12,681 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:33:12,681 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:33:12,681 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:33:12,682 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:33:12,682 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:33:12,682 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:33:12,684 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:33:12,685 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:33:12,694 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:33:12,694 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:33:12,695 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:33:12,704 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:33:12,705 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:33:12,706 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:33:12,715 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:33:12,716 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:33:12,772 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:33:12,775 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:33:12,783 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:33:12,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:33:12,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:33:12,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,794 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,34157,1733844792176, sessionid=0x100105bfa230000, setting cluster-up flag (Was=false)
2024-12-10T15:33:12,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,883 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:33:12,885 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:12,936 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:33:12,937 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,34157,1733844792176
2024-12-10T15:33:12,939 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:33:12,940 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:33:12,941 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:33:12,941 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:33:12,941 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,34157,1733844792176
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:33:12,943 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:12,944 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844822944
2024-12-10T15:33:12,944 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:33:12,944 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:12,945 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:33:12,945 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:33:12,945 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:33:12,946 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:33:12,946 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:33:12,946 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:33:12,946 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844792946,5,FailOnTimeoutGroup]
2024-12-10T15:33:12,946 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844792946,5,FailOnTimeoutGroup]
2024-12-10T15:33:12,946 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:12,946 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:33:12,947 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:12,947 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:12,947 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,947 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:33:12,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:33:12,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:33:12,958 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:33:12,958 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d
2024-12-10T15:33:12,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:33:12,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:33:12,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:12,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:33:12,968 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:33:12,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,968 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:12,968 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:33:12,969 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:33:12,969 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,970 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:12,970 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:33:12,971 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:33:12,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,971 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:12,971 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:33:12,972 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:33:12,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:12,973 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:12,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:33:12,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740
2024-12-10T15:33:12,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740
2024-12-10T15:33:12,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:33:12,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:33:12,975 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:33:12,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:12,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:33:12,978 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:33:12,979 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717910, jitterRate=-0.08713047206401825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:33:12,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844792966Initializing all the Stores at 1733844792966Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844792966Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844792967 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844792967Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844792967Cleaning up temporary data from old regions at 1733844792975 (+8 ms)Region opened successfully at 1733844792979 (+4 ms)
2024-12-10T15:33:12,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:33:12,980 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:33:12,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:33:12,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:33:12,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:33:12,980 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:33:12,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844792980Disabling compacts and flushes for region at 1733844792980Disabling writes for close at 1733844792980Writing region close event to WAL at 1733844792980Closed at 1733844792980
2024-12-10T15:33:12,981 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:33:12,981 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:33:12,981 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:33:12,983 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:33:12,984 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:33:13,032 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(746): ClusterId : b6dbaf2f-b370-495a-8c25-de4af7bd0b71
2024-12-10T15:33:13,032 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:33:13,073 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:33:13,073 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:33:13,084 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:33:13,084 DEBUG [RS:0;d6bcd7308055:36083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da350e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:33:13,096 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:36083
2024-12-10T15:33:13,096 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:33:13,096 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:33:13,096 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:33:13,097 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,34157,1733844792176 with port=36083, startcode=1733844792503
2024-12-10T15:33:13,097 DEBUG [RS:0;d6bcd7308055:36083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:33:13,098 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39535, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:33:13,099 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,099 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34157 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,101 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d
2024-12-10T15:33:13,101 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36495
2024-12-10T15:33:13,101 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:33:13,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:33:13,110 DEBUG [RS:0;d6bcd7308055:36083 {}] zookeeper.ZKUtil(111): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,110 WARN  [RS:0;d6bcd7308055:36083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:33:13,110 INFO  [RS:0;d6bcd7308055:36083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:33:13,110 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,110 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,36083,1733844792503]
2024-12-10T15:33:13,113 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:33:13,115 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:33:13,115 INFO  [RS:0;d6bcd7308055:36083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:33:13,115 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,115 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:33:13,116 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:33:13,116 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,116 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,116 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,116 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:33:13,117 DEBUG [RS:0;d6bcd7308055:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,119 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,36083,1733844792503-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:33:13,134 WARN  [d6bcd7308055:34157 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:33:13,135 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:33:13,135 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,36083,1733844792503-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,136 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,136 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.Replication(171): d6bcd7308055,36083,1733844792503 started
2024-12-10T15:33:13,149 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,149 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,36083,1733844792503, RpcServer on d6bcd7308055/172.17.0.2:36083, sessionid=0x100105bfa230001
2024-12-10T15:33:13,149 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:33:13,149 DEBUG [RS:0;d6bcd7308055:36083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,149 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,36083,1733844792503'
2024-12-10T15:33:13,149 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,36083,1733844792503'
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:33:13,150 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:33:13,151 DEBUG [RS:0;d6bcd7308055:36083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:33:13,151 INFO  [RS:0;d6bcd7308055:36083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:33:13,151 INFO  [RS:0;d6bcd7308055:36083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:33:13,254 INFO  [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C36083%2C1733844792503, suffix=, logDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503, archiveDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs, maxLogs=32
2024-12-10T15:33:13,255 INFO  [RS:0;d6bcd7308055:36083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C36083%2C1733844792503.1733844793254
2024-12-10T15:33:13,263 INFO  [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844793254
2024-12-10T15:33:13,264 DEBUG [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46275:46275),(127.0.0.1/127.0.0.1:43363:43363)]
2024-12-10T15:33:13,385 DEBUG [d6bcd7308055:34157 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:33:13,385 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,387 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,36083,1733844792503, state=OPENING
2024-12-10T15:33:13,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:13,478 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:33:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:33:13,531 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:33:13,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:33:13,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:33:13,531 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,36083,1733844792503}]
2024-12-10T15:33:13,643 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:13,686 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:33:13,688 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37639, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:33:13,693 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:33:13,693 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:33:13,695 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C36083%2C1733844792503.meta, suffix=.meta, logDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503, archiveDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs, maxLogs=32
2024-12-10T15:33:13,695 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C36083%2C1733844792503.meta.1733844793695.meta
2024-12-10T15:33:13,701 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.meta.1733844793695.meta
2024-12-10T15:33:13,717 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46275:46275),(127.0.0.1/127.0.0.1:43363:43363)]
2024-12-10T15:33:13,718 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:33:13,718 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:33:13,718 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:33:13,718 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:33:13,718 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:33:13,718 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:13,719 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:33:13,719 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:33:13,720 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:33:13,720 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:33:13,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:13,721 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:13,721 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:33:13,722 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:33:13,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:13,722 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:13,722 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:33:13,723 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:33:13,723 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:13,723 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:13,723 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:33:13,724 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:33:13,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:13,724 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:33:13,724 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:33:13,725 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740
2024-12-10T15:33:13,726 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740
2024-12-10T15:33:13,727 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:33:13,727 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:33:13,727 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:33:13,729 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:33:13,729 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=808860, jitterRate=0.028519898653030396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:33:13,729 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:33:13,730 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844793719Writing region info on filesystem at 1733844793719Initializing all the Stores at 1733844793719Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844793719Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844793720 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844793720Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844793720Cleaning up temporary data from old regions at 1733844793727 (+7 ms)Running coprocessor post-open hooks at 1733844793729 (+2 ms)Region opened successfully at 1733844793730 (+1 ms)
2024-12-10T15:33:13,731 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844793685
2024-12-10T15:33:13,733 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:33:13,733 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:33:13,734 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,735 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,36083,1733844792503, state=OPEN
2024-12-10T15:33:13,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:33:13,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:33:13,786 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:13,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:33:13,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:33:13,790 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:33:13,790 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,36083,1733844792503 in 255 msec
2024-12-10T15:33:13,794 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:33:13,794 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 809 msec
2024-12-10T15:33:13,795 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:33:13,795 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:33:13,797 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:33:13,797 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,36083,1733844792503, seqNum=-1]
2024-12-10T15:33:13,797 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:33:13,798 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:33:13,804 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 863 msec
2024-12-10T15:33:13,804 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844793804, completionTime=-1
2024-12-10T15:33:13,804 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:33:13,804 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844853806
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844913806
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,806 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,807 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:34157, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,807 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,807 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,808 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:33:13,810 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.232sec
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:33:13,811 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:33:13,813 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:33:13,813 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:33:13,813 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34157,1733844792176-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:13,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b7b3113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:33:13,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,34157,-1 for getting cluster id
2024-12-10T15:33:13,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:33:13,834 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6dbaf2f-b370-495a-8c25-de4af7bd0b71'
2024-12-10T15:33:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:33:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6dbaf2f-b370-495a-8c25-de4af7bd0b71"

2024-12-10T15:33:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3305cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:33:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,34157,-1]
2024-12-10T15:33:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:33:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:33:13,836 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:33:13,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de91bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:33:13,838 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:33:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,36083,1733844792503, seqNum=-1]
2024-12-10T15:33:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:33:13,840 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:33:13,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,34157,1733844792176
2024-12-10T15:33:13,842 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:33:13,845 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:33:13,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T15:33:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d6bcd7308055,34157,1733844792176
2024-12-10T15:33:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a848ad6
2024-12-10T15:33:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T15:33:13,847 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T15:33:13,848 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T15:33:13,848 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T15:33:13,848 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:33:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:13,851 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T15:33:13,851 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:13,851 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-10T15:33:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:33:13,852 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T15:33:13,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741835_1011 (size=405)
2024-12-10T15:33:13,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741835_1011 (size=405)
2024-12-10T15:33:13,862 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d0b5dc39e596e5edf43998a003a3e0f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d
2024-12-10T15:33:13,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741836_1012 (size=88)
2024-12-10T15:33:13,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741836_1012 (size=88)
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 3d0b5dc39e596e5edf43998a003a3e0f, disabling compactions & flushes
2024-12-10T15:33:13,869 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f. after waiting 0 ms
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:13,869 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:13,869 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d0b5dc39e596e5edf43998a003a3e0f:
Waiting for close lock at 1733844793869Disabling compacts and flushes for region at 1733844793869Disabling writes for close at 1733844793869Writing region close event to WAL at 1733844793869Closed at 1733844793869
2024-12-10T15:33:13,871 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T15:33:13,871 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733844793871"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844793871"}]},"ts":"1733844793871"}
2024-12-10T15:33:13,874 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T15:33:13,875 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T15:33:13,875 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844793875"}]},"ts":"1733844793875"}
2024-12-10T15:33:13,878 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-10T15:33:13,878 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=3d0b5dc39e596e5edf43998a003a3e0f, ASSIGN}]
2024-12-10T15:33:13,879 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=3d0b5dc39e596e5edf43998a003a3e0f, ASSIGN
2024-12-10T15:33:13,880 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=3d0b5dc39e596e5edf43998a003a3e0f, ASSIGN; state=OFFLINE, location=d6bcd7308055,36083,1733844792503; forceNewPlan=false, retain=false
2024-12-10T15:33:13,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:13,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698 after 68048ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:33:14,031 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3d0b5dc39e596e5edf43998a003a3e0f, regionState=OPENING, regionLocation=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:14,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=3d0b5dc39e596e5edf43998a003a3e0f, ASSIGN because future has completed
2024-12-10T15:33:14,035 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d0b5dc39e596e5edf43998a003a3e0f, server=d6bcd7308055,36083,1733844792503}]
2024-12-10T15:33:14,193 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:14,194 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d0b5dc39e596e5edf43998a003a3e0f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:33:14,194 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,194 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:33:14,194 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,194 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,196 INFO  [StoreOpener-3d0b5dc39e596e5edf43998a003a3e0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3d0b5dc39e596e5edf43998a003a3e0f 
2024-12-10T15:33:14,197 INFO  [StoreOpener-3d0b5dc39e596e5edf43998a003a3e0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d0b5dc39e596e5edf43998a003a3e0f columnFamilyName info
2024-12-10T15:33:14,197 DEBUG [StoreOpener-3d0b5dc39e596e5edf43998a003a3e0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:33:14,198 INFO  [StoreOpener-3d0b5dc39e596e5edf43998a003a3e0f-1 {}] regionserver.HStore(327): Store=3d0b5dc39e596e5edf43998a003a3e0f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:33:14,198 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,199 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,200 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,200 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,200 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,202 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,213 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:33:14,214 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3d0b5dc39e596e5edf43998a003a3e0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802792, jitterRate=0.020803436636924744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:33:14,214 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:33:14,214 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3d0b5dc39e596e5edf43998a003a3e0f:
Running coprocessor pre-open hook at 1733844794194Writing region info on filesystem at 1733844794194Initializing all the Stores at 1733844794195 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844794195Cleaning up temporary data from old regions at 1733844794200 (+5 ms)Running coprocessor post-open hooks at 1733844794214 (+14 ms)Region opened successfully at 1733844794214
2024-12-10T15:33:14,215 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f., pid=6, masterSystemTime=1733844794188
2024-12-10T15:33:14,218 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:14,218 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:14,219 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3d0b5dc39e596e5edf43998a003a3e0f, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,36083,1733844792503
2024-12-10T15:33:14,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d0b5dc39e596e5edf43998a003a3e0f, server=d6bcd7308055,36083,1733844792503 because future has completed
2024-12-10T15:33:14,226 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T15:33:14,226 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d0b5dc39e596e5edf43998a003a3e0f, server=d6bcd7308055,36083,1733844792503 in 188 msec
2024-12-10T15:33:14,229 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T15:33:14,229 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=3d0b5dc39e596e5edf43998a003a3e0f, ASSIGN in 348 msec
2024-12-10T15:33:14,230 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T15:33:14,230 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844794230"}]},"ts":"1733844794230"}
2024-12-10T15:33:14,232 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-10T15:33:14,233 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T15:33:14,235 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 385 msec
2024-12-10T15:33:14,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:14,644 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:14,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:15,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:15,644 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:15,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:16,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:16,645 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:16,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:16,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:16,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:16,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:16,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:16,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:16,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,003 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,003 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,004 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,004 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,004 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,004 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:17,515 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:33:17,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,545 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,545 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,545 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,549 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,553 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:33:17,646 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:17,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:18,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:18,646 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:18,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:19,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T15:33:19,114 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-10T15:33:19,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:19,647 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:19,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:20,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:20,648 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:20,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:21,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:21,648 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:21,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:21,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T15:33:21,986 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T15:33:21,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:33:21,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T15:33:21,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T15:33:21,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-10T15:33:21,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:21,988 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T15:33:22,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:22,649 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:22,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:23,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:23,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta after 68058ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:33:23,650 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:33:23,951 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T15:33:23,951 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-10T15:33:23,955 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:23,955 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f., hostname=d6bcd7308055,36083,1733844792503, seqNum=2]
2024-12-10T15:33:23,968 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:23,974 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T15:33:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-10T15:33:23,975 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T15:33:23,976 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T15:33:23,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:24,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36083 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-10T15:33:24,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:24,141 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 3d0b5dc39e596e5edf43998a003a3e0f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T15:33:24,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/731d19e181814b1d8a6f9b7481c18be5 is 1080, key is row0001/info:/1733844803961/Put/seqid=0
2024-12-10T15:33:24,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741837_1013 (size=6033)
2024-12-10T15:33:24,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741837_1013 (size=6033)
2024-12-10T15:33:24,171 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/731d19e181814b1d8a6f9b7481c18be5
2024-12-10T15:33:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/731d19e181814b1d8a6f9b7481c18be5 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5
2024-12-10T15:33:24,185 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5, entries=1, sequenceid=5, filesize=5.9 K
2024-12-10T15:33:24,186 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 46ms, sequenceid=5, compaction requested=false
2024-12-10T15:33:24,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3d0b5dc39e596e5edf43998a003a3e0f:

2024-12-10T15:33:24,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:24,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-10T15:33:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-10T15:33:24,194 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-10T15:33:24,194 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 215 msec
2024-12-10T15:33:24,196 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 225 msec
2024-12-10T15:33:24,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:24,650 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:24,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:25,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:25,651 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:25,984 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:26,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:26,651 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:26,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:27,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:27,652 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:27,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:28,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:28,653 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:28,653 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295 after 68044ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T15:33:28,986 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:29,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:29,654 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:29,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:30,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:30,655 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:30,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:31,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:31,655 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:31,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:32,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:32,656 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:32,989 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:33,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:33,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:33,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-10T15:33:34,020 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T15:33:34,023 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-10T15:33:34,025 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T15:33:34,026 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T15:33:34,026 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T15:33:34,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36083 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-10T15:33:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:34,181 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3d0b5dc39e596e5edf43998a003a3e0f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T15:33:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/3863912b0ec84ff69a11b2ea9b2ae8f5 is 1080, key is row0002/info:/1733844814021/Put/seqid=0
2024-12-10T15:33:34,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741838_1014 (size=6033)
2024-12-10T15:33:34,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741838_1014 (size=6033)
2024-12-10T15:33:34,191 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/3863912b0ec84ff69a11b2ea9b2ae8f5
2024-12-10T15:33:34,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/3863912b0ec84ff69a11b2ea9b2ae8f5 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5
2024-12-10T15:33:34,204 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5, entries=1, sequenceid=9, filesize=5.9 K
2024-12-10T15:33:34,205 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 24ms, sequenceid=9, compaction requested=false
2024-12-10T15:33:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3d0b5dc39e596e5edf43998a003a3e0f:

2024-12-10T15:33:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-10T15:33:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-10T15:33:34,210 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-10T15:33:34,210 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 181 msec
2024-12-10T15:33:34,212 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec
2024-12-10T15:33:34,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:34,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:34,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:35,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:35,658 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:35,991 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:36,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:36,659 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:36,991 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:37,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:37,660 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:37,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:38,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:38,661 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:38,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:39,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:39,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:39,995 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:40,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:40,664 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:40,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:41,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:41,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:41,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:42,131 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:33:42,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:42,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:42,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:43,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:43,666 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:43,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-10T15:33:44,060 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T15:33:44,063 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C36083%2C1733844792503.1733844824063
2024-12-10T15:33:44,068 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:44,068 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:44,068 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:44,068 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:44,068 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:44,068 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844793254 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844824063
2024-12-10T15:33:44,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43363:43363),(127.0.0.1/127.0.0.1:46275:46275)]
2024-12-10T15:33:44,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844793254 is not closed yet, will try archiving it next time
2024-12-10T15:33:44,070 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:44,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741833_1009 (size=5546)
2024-12-10T15:33:44,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741833_1009 (size=5546)
2024-12-10T15:33:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:44,072 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T15:33:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-10T15:33:44,073 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T15:33:44,074 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T15:33:44,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36083 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-10T15:33:44,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:44,237 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3d0b5dc39e596e5edf43998a003a3e0f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T15:33:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/ab3bfde991b44d77a1894470bc2a3a0e is 1080, key is row0003/info:/1733844824061/Put/seqid=0
2024-12-10T15:33:44,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741840_1016 (size=6033)
2024-12-10T15:33:44,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741840_1016 (size=6033)
2024-12-10T15:33:44,246 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/ab3bfde991b44d77a1894470bc2a3a0e
2024-12-10T15:33:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/ab3bfde991b44d77a1894470bc2a3a0e as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e
2024-12-10T15:33:44,259 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e, entries=1, sequenceid=13, filesize=5.9 K
2024-12-10T15:33:44,260 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 23ms, sequenceid=13, compaction requested=true
2024-12-10T15:33:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3d0b5dc39e596e5edf43998a003a3e0f:

2024-12-10T15:33:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-10T15:33:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-10T15:33:44,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-10T15:33:44,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 189 msec
2024-12-10T15:33:44,268 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-12-10T15:33:44,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:44,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:44,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:45,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:45,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:45,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:46,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:46,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:47,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:47,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:47,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:48,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:48,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:48,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:49,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:49,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:49,670 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:50,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:50,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:50,670 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:51,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:51,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:51,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:52,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:52,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:52,672 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:53,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:53,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:53,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:53,948 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T15:33:53,948 INFO  [master/d6bcd7308055:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T15:33:54,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-10T15:33:54,090 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T15:33:54,090 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:33:54,091 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:33:54,091 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3d0b5dc39e596e5edf43998a003a3e0f/info is initiating minor compaction (all files)
2024-12-10T15:33:54,092 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:33:54,092 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:33:54,092 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3d0b5dc39e596e5edf43998a003a3e0f/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:54,092 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e] into tmpdir=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp, totalSize=17.7 K
2024-12-10T15:33:54,092 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 731d19e181814b1d8a6f9b7481c18be5, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733844803961
2024-12-10T15:33:54,093 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3863912b0ec84ff69a11b2ea9b2ae8f5, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733844814021
2024-12-10T15:33:54,093 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ab3bfde991b44d77a1894470bc2a3a0e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733844824061
2024-12-10T15:33:54,103 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3d0b5dc39e596e5edf43998a003a3e0f#info#compaction#46 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:33:54,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/42ca483d42364767a13066b19d6eb095 is 1080, key is row0001/info:/1733844803961/Put/seqid=0
2024-12-10T15:33:54,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741841_1017 (size=8296)
2024-12-10T15:33:54,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741841_1017 (size=8296)
2024-12-10T15:33:54,116 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/42ca483d42364767a13066b19d6eb095 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/42ca483d42364767a13066b19d6eb095
2024-12-10T15:33:54,123 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d0b5dc39e596e5edf43998a003a3e0f/info of 3d0b5dc39e596e5edf43998a003a3e0f into 42ca483d42364767a13066b19d6eb095(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:33:54,123 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3d0b5dc39e596e5edf43998a003a3e0f:

2024-12-10T15:33:54,126 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C36083%2C1733844792503.1733844834126
2024-12-10T15:33:54,132 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:54,132 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:54,132 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:54,132 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:54,132 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:33:54,133 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844824063 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844834126
2024-12-10T15:33:54,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46275:46275),(127.0.0.1/127.0.0.1:43363:43363)]
2024-12-10T15:33:54,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844824063 is not closed yet, will try archiving it next time
2024-12-10T15:33:54,134 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844793254 to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs/d6bcd7308055%2C36083%2C1733844792503.1733844793254
2024-12-10T15:33:54,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741839_1015 (size=2520)
2024-12-10T15:33:54,135 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:54,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741839_1015 (size=2520)
2024-12-10T15:33:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:33:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-10T15:33:54,137 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T15:33:54,139 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T15:33:54,139 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T15:33:54,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36083 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-10T15:33:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:54,293 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3d0b5dc39e596e5edf43998a003a3e0f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T15:33:54,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/f16b9353cc8143a6b51b153f4664a41f is 1080, key is row0000/info:/1733844834124/Put/seqid=0
2024-12-10T15:33:54,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741843_1019 (size=6033)
2024-12-10T15:33:54,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741843_1019 (size=6033)
2024-12-10T15:33:54,307 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/f16b9353cc8143a6b51b153f4664a41f
2024-12-10T15:33:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/f16b9353cc8143a6b51b153f4664a41f as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/f16b9353cc8143a6b51b153f4664a41f
2024-12-10T15:33:54,323 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/f16b9353cc8143a6b51b153f4664a41f, entries=1, sequenceid=18, filesize=5.9 K
2024-12-10T15:33:54,324 INFO  [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 32ms, sequenceid=18, compaction requested=false
2024-12-10T15:33:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3d0b5dc39e596e5edf43998a003a3e0f:

2024-12-10T15:33:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:33:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-10T15:33:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-10T15:33:54,329 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-10T15:33:54,329 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-12-10T15:33:54,332 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 195 msec
2024-12-10T15:33:54,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:54,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:55,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:55,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:55,674 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:56,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:56,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:56,675 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:57,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:57,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:57,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:58,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:58,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:58,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:59,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:59,194 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 3d0b5dc39e596e5edf43998a003a3e0f, had cached 0 bytes from a total of 14329
2024-12-10T15:33:59,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:33:59,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:00,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:00,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:00,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:01,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:01,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:01,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:02,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:02,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:02,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:03,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:03,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:03,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:04,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-10T15:34:04,151 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T15:34:04,157 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C36083%2C1733844792503.1733844844157
2024-12-10T15:34:04,166 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,167 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,167 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,167 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,167 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,167 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844834126 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844844157
2024-12-10T15:34:04,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46275:46275),(127.0.0.1/127.0.0.1:43363:43363)]
2024-12-10T15:34:04,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844834126 is not closed yet, will try archiving it next time
2024-12-10T15:34:04,168 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:34:04,168 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/WALs/d6bcd7308055,36083,1733844792503/d6bcd7308055%2C36083%2C1733844792503.1733844824063 to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs/d6bcd7308055%2C36083%2C1733844792503.1733844824063
2024-12-10T15:34:04,169 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:04,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:04,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:04,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:04,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741842_1018 (size=2026)
2024-12-10T15:34:04,169 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:34:04,169 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=342693364, stopped=false
2024-12-10T15:34:04,169 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,34157,1733844792176
2024-12-10T15:34:04,169 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:34:04,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741842_1018 (size=2026)
2024-12-10T15:34:04,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:04,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:04,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:04,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:04,228 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:04,228 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:04,228 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:04,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:04,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:04,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:04,229 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,36083,1733844792503' *****
2024-12-10T15:34:04,229 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:34:04,230 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:34:04,230 INFO  [RS:0;d6bcd7308055:36083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:34:04,230 INFO  [RS:0;d6bcd7308055:36083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:34:04,231 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:34:04,231 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:34:04,231 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,36083,1733844792503
2024-12-10T15:34:04,231 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:04,231 INFO  [RS:0;d6bcd7308055:36083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:36083.
2024-12-10T15:34:04,232 DEBUG [RS:0;d6bcd7308055:36083 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:04,232 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3d0b5dc39e596e5edf43998a003a3e0f, disabling compactions & flushes
2024-12-10T15:34:04,232 DEBUG [RS:0;d6bcd7308055:36083 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:04,232 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:34:04,232 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:34:04,232 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:34:04,232 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:34:04,232 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:34:04,232 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:34:04,232 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f. after waiting 0 ms
2024-12-10T15:34:04,232 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:34:04,232 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3d0b5dc39e596e5edf43998a003a3e0f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T15:34:04,233 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T15:34:04,233 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3d0b5dc39e596e5edf43998a003a3e0f=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.}
2024-12-10T15:34:04,233 DEBUG [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3d0b5dc39e596e5edf43998a003a3e0f
2024-12-10T15:34:04,233 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:34:04,233 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:34:04,233 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:34:04,233 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:34:04,233 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:34:04,234 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-10T15:34:04,238 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/676749bbee704e259aaae3304ae4ed0c is 1080, key is row0001/info:/1733844844154/Put/seqid=0
2024-12-10T15:34:04,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741845_1021 (size=6033)
2024-12-10T15:34:04,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741845_1021 (size=6033)
2024-12-10T15:34:04,243 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/676749bbee704e259aaae3304ae4ed0c
2024-12-10T15:34:04,248 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/.tmp/info/676749bbee704e259aaae3304ae4ed0c as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/676749bbee704e259aaae3304ae4ed0c
2024-12-10T15:34:04,250 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/info/a5fb6019fc154a729299f64993916981 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f./info:regioninfo/1733844794219/Put/seqid=0
2024-12-10T15:34:04,254 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/676749bbee704e259aaae3304ae4ed0c, entries=1, sequenceid=22, filesize=5.9 K
2024-12-10T15:34:04,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741846_1022 (size=7308)
2024-12-10T15:34:04,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741846_1022 (size=7308)
2024-12-10T15:34:04,256 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/info/a5fb6019fc154a729299f64993916981
2024-12-10T15:34:04,257 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 25ms, sequenceid=22, compaction requested=true
2024-12-10T15:34:04,258 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e] to archive
2024-12-10T15:34:04,259 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:34:04,260 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5 to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/731d19e181814b1d8a6f9b7481c18be5
2024-12-10T15:34:04,261 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5 to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/3863912b0ec84ff69a11b2ea9b2ae8f5
2024-12-10T15:34:04,261 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e to hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/info/ab3bfde991b44d77a1894470bc2a3a0e
2024-12-10T15:34:04,261 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d6bcd7308055:34157 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T15:34:04,261 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [731d19e181814b1d8a6f9b7481c18be5=6033, 3863912b0ec84ff69a11b2ea9b2ae8f5=6033, ab3bfde991b44d77a1894470bc2a3a0e=6033]
2024-12-10T15:34:04,264 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/3d0b5dc39e596e5edf43998a003a3e0f/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-10T15:34:04,265 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:34:04,265 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3d0b5dc39e596e5edf43998a003a3e0f:
Waiting for close lock at 1733844844232Running coprocessor pre-close hooks at 1733844844232Disabling compacts and flushes for region at 1733844844232Disabling writes for close at 1733844844232Obtaining lock to block concurrent updates at 1733844844232Preparing flush snapshotting stores in 3d0b5dc39e596e5edf43998a003a3e0f at 1733844844232Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733844844233 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f. at 1733844844235 (+2 ms)Flushing 3d0b5dc39e596e5edf43998a003a3e0f/info: creating writer at 1733844844235Flushing 3d0b5dc39e596e5edf43998a003a3e0f/info: appending metadata at 1733844844238 (+3 ms)Flushing 3d0b5dc39e596e5edf43998a003a3e0f/info: closing flushed file at 1733844844238Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16cc3b5c: reopening flushed file at 1733844844248 (+10 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3d0b5dc39e596e5edf43998a003a3e0f in 25ms, sequenceid=22, compaction requested=true at 1733844844257 (+9 ms)Writing region close event to WAL at 1733844844262 (+5 ms)Running coprocessor post-close hooks at 1733844844265 (+3 ms)Closed at 1733844844265
2024-12-10T15:34:04,265 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733844793848.3d0b5dc39e596e5edf43998a003a3e0f.
2024-12-10T15:34:04,274 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/ns/87f41680d39a494e95ac477ffae54760 is 43, key is default/ns:d/1733844793799/Put/seqid=0
2024-12-10T15:34:04,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741847_1023 (size=5153)
2024-12-10T15:34:04,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741847_1023 (size=5153)
2024-12-10T15:34:04,278 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/ns/87f41680d39a494e95ac477ffae54760
2024-12-10T15:34:04,296 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/table/f2dcfd8b386248cd9813b7886b458021 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733844794230/Put/seqid=0
2024-12-10T15:34:04,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741848_1024 (size=5508)
2024-12-10T15:34:04,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741848_1024 (size=5508)
2024-12-10T15:34:04,301 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/table/f2dcfd8b386248cd9813b7886b458021
2024-12-10T15:34:04,306 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/info/a5fb6019fc154a729299f64993916981 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/info/a5fb6019fc154a729299f64993916981
2024-12-10T15:34:04,311 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/info/a5fb6019fc154a729299f64993916981, entries=10, sequenceid=11, filesize=7.1 K
2024-12-10T15:34:04,312 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/ns/87f41680d39a494e95ac477ffae54760 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/ns/87f41680d39a494e95ac477ffae54760
2024-12-10T15:34:04,317 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/ns/87f41680d39a494e95ac477ffae54760, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T15:34:04,318 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/.tmp/table/f2dcfd8b386248cd9813b7886b458021 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/table/f2dcfd8b386248cd9813b7886b458021
2024-12-10T15:34:04,323 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/table/f2dcfd8b386248cd9813b7886b458021, entries=2, sequenceid=11, filesize=5.4 K
2024-12-10T15:34:04,324 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false
2024-12-10T15:34:04,328 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T15:34:04,329 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:34:04,329 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:04,329 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844844233Running coprocessor pre-close hooks at 1733844844233Disabling compacts and flushes for region at 1733844844233Disabling writes for close at 1733844844233Obtaining lock to block concurrent updates at 1733844844234 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733844844234Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733844844234Flushing stores of hbase:meta,,1.1588230740 at 1733844844236 (+2 ms)Flushing 1588230740/info: creating writer at 1733844844236Flushing 1588230740/info: appending metadata at 1733844844250 (+14 ms)Flushing 1588230740/info: closing flushed file at 1733844844250Flushing 1588230740/ns: creating writer at 1733844844261 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733844844273 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733844844273Flushing 1588230740/table: creating writer at 1733844844282 (+9 ms)Flushing 1588230740/table: appending metadata at 1733844844295 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733844844296 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55b976b4: reopening flushed file at 1733844844305 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d178e5f: reopening flushed file at 1733844844312 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264bfdb5: reopening flushed file at 1733844844317 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false at 1733844844324 (+7 ms)Writing region close event to WAL at 1733844844325 (+1 ms)Running coprocessor post-close hooks at 1733844844329 (+4 ms)Closed at 1733844844329
2024-12-10T15:34:04,329 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:04,433 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,36083,1733844792503; all regions closed.
2024-12-10T15:34:04,434 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,434 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,434 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,434 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,434 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741834_1010 (size=3306)
2024-12-10T15:34:04,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741834_1010 (size=3306)
2024-12-10T15:34:04,440 DEBUG [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs
2024-12-10T15:34:04,440 INFO  [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C36083%2C1733844792503.meta:.meta(num 1733844793695)
2024-12-10T15:34:04,441 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,441 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,441 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,441 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,442 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741844_1020 (size=1252)
2024-12-10T15:34:04,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741844_1020 (size=1252)
2024-12-10T15:34:04,449 DEBUG [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/oldWALs
2024-12-10T15:34:04,449 INFO  [RS:0;d6bcd7308055:36083 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C36083%2C1733844792503:(num 1733844844157)
2024-12-10T15:34:04,449 DEBUG [RS:0;d6bcd7308055:36083 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:04,449 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:34:04,449 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:04,449 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:04,450 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:04,450 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:04,450 INFO  [RS:0;d6bcd7308055:36083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36083
2024-12-10T15:34:04,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,36083,1733844792503
2024-12-10T15:34:04,458 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:04,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:34:04,469 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,36083,1733844792503]
2024-12-10T15:34:04,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:04,553 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,36083,1733844792503 already deleted, retry=false
2024-12-10T15:34:04,553 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,36083,1733844792503 expired; onlineServers=0
2024-12-10T15:34:04,553 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,34157,1733844792176' *****
2024-12-10T15:34:04,553 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:34:04,553 INFO  [M:0;d6bcd7308055:34157 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:04,553 INFO  [M:0;d6bcd7308055:34157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:04,553 DEBUG [M:0;d6bcd7308055:34157 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:34:04,553 DEBUG [M:0;d6bcd7308055:34157 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:34:04,553 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:34:04,553 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844792946 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844792946,5,FailOnTimeoutGroup]
2024-12-10T15:34:04,553 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844792946 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844792946,5,FailOnTimeoutGroup]
2024-12-10T15:34:04,554 INFO  [M:0;d6bcd7308055:34157 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:04,554 INFO  [M:0;d6bcd7308055:34157 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:04,554 DEBUG [M:0;d6bcd7308055:34157 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:34:04,554 INFO  [M:0;d6bcd7308055:34157 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:34:04,554 INFO  [M:0;d6bcd7308055:34157 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:04,554 INFO  [M:0;d6bcd7308055:34157 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:34:04,554 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:34:04,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:34:04,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:04,564 DEBUG [M:0;d6bcd7308055:34157 {}] zookeeper.ZKUtil(347): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T15:34:04,564 WARN  [M:0;d6bcd7308055:34157 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T15:34:04,564 INFO  [M:0;d6bcd7308055:34157 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/.lastflushedseqids
2024-12-10T15:34:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:04,569 INFO  [RS:0;d6bcd7308055:36083 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x100105bfa230001, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:04,569 INFO  [RS:0;d6bcd7308055:36083 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,36083,1733844792503; zookeeper connection closed.
2024-12-10T15:34:04,570 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@72167c44 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@72167c44
2024-12-10T15:34:04,570 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:34:04,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741849_1025 (size=130)
2024-12-10T15:34:04,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741849_1025 (size=130)
2024-12-10T15:34:04,574 INFO  [M:0;d6bcd7308055:34157 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:34:04,574 INFO  [M:0;d6bcd7308055:34157 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:34:04,575 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:34:04,575 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:04,575 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:04,575 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:34:04,575 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:04,575 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.54 KB heapSize=54.89 KB
2024-12-10T15:34:04,591 DEBUG [M:0;d6bcd7308055:34157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/470ce6e1c21043f094c4e757f9c690b9 is 82, key is hbase:meta,,1/info:regioninfo/1733844793734/Put/seqid=0
2024-12-10T15:34:04,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741850_1026 (size=5672)
2024-12-10T15:34:04,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741850_1026 (size=5672)
2024-12-10T15:34:04,596 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/470ce6e1c21043f094c4e757f9c690b9
2024-12-10T15:34:04,614 DEBUG [M:0;d6bcd7308055:34157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df965e5138384e9588059e18341537aa is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733844794235/Put/seqid=0
2024-12-10T15:34:04,619 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741851_1027 (size=7817)
2024-12-10T15:34:04,619 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741851_1027 (size=7817)
2024-12-10T15:34:04,619 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.94 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df965e5138384e9588059e18341537aa
2024-12-10T15:34:04,623 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df965e5138384e9588059e18341537aa
2024-12-10T15:34:04,637 DEBUG [M:0;d6bcd7308055:34157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4503b26e8fb648379b64c2f226eba6bf is 69, key is d6bcd7308055,36083,1733844792503/rs:state/1733844793099/Put/seqid=0
2024-12-10T15:34:04,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741852_1028 (size=5156)
2024-12-10T15:34:04,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741852_1028 (size=5156)
2024-12-10T15:34:04,642 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4503b26e8fb648379b64c2f226eba6bf
2024-12-10T15:34:04,659 DEBUG [M:0;d6bcd7308055:34157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/88474c0fef6d4838a887fa6b968b3f43 is 52, key is load_balancer_on/state:d/1733844793844/Put/seqid=0
2024-12-10T15:34:04,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741853_1029 (size=5056)
2024-12-10T15:34:04,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741853_1029 (size=5056)
2024-12-10T15:34:04,664 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/88474c0fef6d4838a887fa6b968b3f43
2024-12-10T15:34:04,668 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/470ce6e1c21043f094c4e757f9c690b9 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/470ce6e1c21043f094c4e757f9c690b9
2024-12-10T15:34:04,673 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/470ce6e1c21043f094c4e757f9c690b9, entries=8, sequenceid=121, filesize=5.5 K
2024-12-10T15:34:04,674 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df965e5138384e9588059e18341537aa as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df965e5138384e9588059e18341537aa
2024-12-10T15:34:04,679 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df965e5138384e9588059e18341537aa
2024-12-10T15:34:04,679 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df965e5138384e9588059e18341537aa, entries=14, sequenceid=121, filesize=7.6 K
2024-12-10T15:34:04,680 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4503b26e8fb648379b64c2f226eba6bf as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4503b26e8fb648379b64c2f226eba6bf
2024-12-10T15:34:04,681 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:04,685 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4503b26e8fb648379b64c2f226eba6bf, entries=1, sequenceid=121, filesize=5.0 K
2024-12-10T15:34:04,686 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/88474c0fef6d4838a887fa6b968b3f43 as hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/88474c0fef6d4838a887fa6b968b3f43
2024-12-10T15:34:04,691 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36495/user/jenkins/test-data/875d6650-49c2-33b1-d845-3d6bd5a17a9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/88474c0fef6d4838a887fa6b968b3f43, entries=1, sequenceid=121, filesize=4.9 K
2024-12-10T15:34:04,692 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false
2024-12-10T15:34:04,695 INFO  [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:04,696 DEBUG [M:0;d6bcd7308055:34157 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844844575Disabling compacts and flushes for region at 1733844844575Disabling writes for close at 1733844844575Obtaining lock to block concurrent updates at 1733844844575Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844844575Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44584, getHeapSize=56144, getOffHeapSize=0, getCellsCount=140 at 1733844844576 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844844576Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844844576Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844844591 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844844591Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844844601 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844844614 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844844614Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844844623 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844844636 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844844636Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844844646 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844844658 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844844658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27e3e962: reopening flushed file at 1733844844668 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bccd8d6: reopening flushed file at 1733844844673 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ac7b758: reopening flushed file at 1733844844679 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@121de586: reopening flushed file at 1733844844685 (+6 ms)Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false at 1733844844692 (+7 ms)Writing region close event to WAL at 1733844844695 (+3 ms)Closed at 1733844844695
2024-12-10T15:34:04,696 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,696 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,696 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,696 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,696 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:04,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40271 is added to blk_1073741830_1006 (size=52981)
2024-12-10T15:34:04,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39869 is added to blk_1073741830_1006 (size=52981)
2024-12-10T15:34:04,699 INFO  [M:0;d6bcd7308055:34157 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:34:04,699 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:04,699 INFO  [M:0;d6bcd7308055:34157 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34157
2024-12-10T15:34:04,699 INFO  [M:0;d6bcd7308055:34157 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:04,811 INFO  [M:0;d6bcd7308055:34157 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:04,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:04,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34157-0x100105bfa230000, quorum=127.0.0.1:58015, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:04,814 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ca8115c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:04,814 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7fd5b514{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:04,814 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:04,814 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@796d466{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:04,814 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@453c7069{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:04,815 WARN  [BP-668363613-172.17.0.2-1733844789113 heartbeating to localhost/127.0.0.1:36495 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:04,815 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:04,815 WARN  [BP-668363613-172.17.0.2-1733844789113 heartbeating to localhost/127.0.0.1:36495 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-668363613-172.17.0.2-1733844789113 (Datanode Uuid 1302ef6d-1452-4d94-9aa5-f4bc642d3123) service to localhost/127.0.0.1:36495
2024-12-10T15:34:04,815 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:04,816 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data3/current/BP-668363613-172.17.0.2-1733844789113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:04,816 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data4/current/BP-668363613-172.17.0.2-1733844789113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:04,816 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:04,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@795929af{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:04,818 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@23e7edd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:04,818 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:04,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9590158{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:04,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bc13808{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:04,819 WARN  [BP-668363613-172.17.0.2-1733844789113 heartbeating to localhost/127.0.0.1:36495 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:04,819 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:04,819 WARN  [BP-668363613-172.17.0.2-1733844789113 heartbeating to localhost/127.0.0.1:36495 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-668363613-172.17.0.2-1733844789113 (Datanode Uuid e4710c35-46f9-468d-8223-7f18419cbc0c) service to localhost/127.0.0.1:36495
2024-12-10T15:34:04,819 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:04,820 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data1/current/BP-668363613-172.17.0.2-1733844789113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:04,820 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/cluster_9e330e6d-aa7f-29cd-4dfb-6dac48748090/data/data2/current/BP-668363613-172.17.0.2-1733844789113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:04,820 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:04,825 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f209b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:34:04,825 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@489be359{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:04,825 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:04,825 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ac313e8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:04,825 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c518279{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:04,833 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:34:04,849 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:34:04,855 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=212 (was 183)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-13
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/d6bcd7308055:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:36495 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:36495 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:36495
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-12
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:36495 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-14
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36495
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=7265 (was 6476) - AvailableMemoryMB LEAK? -
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=212, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=7265
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.log.dir so I do NOT create it in target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a39b097-5eed-156b-eb2f-984fff270b10/hadoop.tmp.dir so I do NOT create it in target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e, deleteOnExit=true
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/test.cache.data in system properties and HBase conf
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:34:04,862 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:34:04,863 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:34:04,863 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:34:04,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:34:04,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:34:04,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:34:04,875 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:34:05,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:05,121 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:34:05,235 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:05,238 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:05,239 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:05,239 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:05,239 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:34:05,240 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:05,240 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e2d8e1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:05,240 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32e62001{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:05,344 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4b7ab4bd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/java.io.tmpdir/jetty-localhost-33687-hadoop-hdfs-3_4_1-tests_jar-_-any-6984182600264395151/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:34:05,344 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2fcc97ec{HTTP/1.1, (http/1.1)}{localhost:33687}
2024-12-10T15:34:05,344 INFO  [Time-limited test {}] server.Server(415): Started @259935ms
2024-12-10T15:34:05,354 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:34:05,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:05,659 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:05,662 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:05,662 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:05,662 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:05,662 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:34:05,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fdb83bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:05,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42a7c471{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:05,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:05,750 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58368375{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/java.io.tmpdir/jetty-localhost-40049-hadoop-hdfs-3_4_1-tests_jar-_-any-10412224396621115219/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:05,750 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@338baa7b{HTTP/1.1, (http/1.1)}{localhost:40049}
2024-12-10T15:34:05,751 INFO  [Time-limited test {}] server.Server(415): Started @260341ms
2024-12-10T15:34:05,752 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:34:05,775 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:05,778 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:05,779 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:05,779 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:05,779 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:34:05,779 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@426b898{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:05,780 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@206e2def{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:05,869 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25d8f267{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/java.io.tmpdir/jetty-localhost-43573-hadoop-hdfs-3_4_1-tests_jar-_-any-13284073573227756211/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:05,870 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7aa18c0f{HTTP/1.1, (http/1.1)}{localhost:43573}
2024-12-10T15:34:05,870 INFO  [Time-limited test {}] server.Server(415): Started @260460ms
2024-12-10T15:34:05,870 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:34:06,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:06,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:06,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:06,929 WARN  [Thread-1977 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data2/current/BP-1146057193-172.17.0.2-1733844844878/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:06,929 WARN  [Thread-1976 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data1/current/BP-1146057193-172.17.0.2-1733844844878/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:06,947 WARN  [Thread-1940 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:34:06,949 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc14e177023e5b75a with lease ID 0xadeae9e525a84044: Processing first storage report for DS-5286bb13-6082-487c-a91e-47105e35ad5e from datanode DatanodeRegistration(127.0.0.1:45095, datanodeUuid=cd57e13b-228c-44a3-a7d1-01424a7febbb, infoPort=39849, infoSecurePort=0, ipcPort=46383, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878)
2024-12-10T15:34:06,949 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc14e177023e5b75a with lease ID 0xadeae9e525a84044: from storage DS-5286bb13-6082-487c-a91e-47105e35ad5e node DatanodeRegistration(127.0.0.1:45095, datanodeUuid=cd57e13b-228c-44a3-a7d1-01424a7febbb, infoPort=39849, infoSecurePort=0, ipcPort=46383, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:06,949 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc14e177023e5b75a with lease ID 0xadeae9e525a84044: Processing first storage report for DS-bf07a9b2-6d71-4378-a925-77985cc1faef from datanode DatanodeRegistration(127.0.0.1:45095, datanodeUuid=cd57e13b-228c-44a3-a7d1-01424a7febbb, infoPort=39849, infoSecurePort=0, ipcPort=46383, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878)
2024-12-10T15:34:06,949 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc14e177023e5b75a with lease ID 0xadeae9e525a84044: from storage DS-bf07a9b2-6d71-4378-a925-77985cc1faef node DatanodeRegistration(127.0.0.1:45095, datanodeUuid=cd57e13b-228c-44a3-a7d1-01424a7febbb, infoPort=39849, infoSecurePort=0, ipcPort=46383, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:07,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:07,064 WARN  [Thread-1988 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data4/current/BP-1146057193-172.17.0.2-1733844844878/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:07,064 WARN  [Thread-1987 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data3/current/BP-1146057193-172.17.0.2-1733844844878/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:07,082 WARN  [Thread-1963 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:34:07,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4614973942d3e6cc with lease ID 0xadeae9e525a84045: Processing first storage report for DS-71d158e4-e325-4ed8-946d-7f78fa5df37c from datanode DatanodeRegistration(127.0.0.1:46355, datanodeUuid=fbdd1eb9-c22e-4500-9333-5668f2fc13e8, infoPort=42873, infoSecurePort=0, ipcPort=36033, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878)
2024-12-10T15:34:07,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4614973942d3e6cc with lease ID 0xadeae9e525a84045: from storage DS-71d158e4-e325-4ed8-946d-7f78fa5df37c node DatanodeRegistration(127.0.0.1:46355, datanodeUuid=fbdd1eb9-c22e-4500-9333-5668f2fc13e8, infoPort=42873, infoSecurePort=0, ipcPort=36033, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:07,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4614973942d3e6cc with lease ID 0xadeae9e525a84045: Processing first storage report for DS-0ffd4006-874d-4fe6-861f-ab7c928e1a11 from datanode DatanodeRegistration(127.0.0.1:46355, datanodeUuid=fbdd1eb9-c22e-4500-9333-5668f2fc13e8, infoPort=42873, infoSecurePort=0, ipcPort=36033, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878)
2024-12-10T15:34:07,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4614973942d3e6cc with lease ID 0xadeae9e525a84045: from storage DS-0ffd4006-874d-4fe6-861f-ab7c928e1a11 node DatanodeRegistration(127.0.0.1:46355, datanodeUuid=fbdd1eb9-c22e-4500-9333-5668f2fc13e8, infoPort=42873, infoSecurePort=0, ipcPort=36033, storageInfo=lv=-57;cid=testClusterID;nsid=333587829;c=1733844844878), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:07,100 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b
2024-12-10T15:34:07,102 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/zookeeper_0, clientPort=64316, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:34:07,103 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64316
2024-12-10T15:34:07,104 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,105 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:34:07,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:34:07,115 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3 with version=8
2024-12-10T15:34:07,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:34:07,117 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:34:07,117 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,117 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:34:07,118 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35351
2024-12-10T15:34:07,119 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35351 connecting to ZooKeeper ensemble=127.0.0.1:64316
2024-12-10T15:34:07,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353510x0, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:34:07,180 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35351-0x100105cd0c40000 connected
2024-12-10T15:34:07,261 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,265 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:07,270 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3, hbase.cluster.distributed=false
2024-12-10T15:34:07,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:34:07,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35351
2024-12-10T15:34:07,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35351
2024-12-10T15:34:07,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35351
2024-12-10T15:34:07,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35351
2024-12-10T15:34:07,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35351
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:34:07,291 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:34:07,292 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44815
2024-12-10T15:34:07,293 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44815 connecting to ZooKeeper ensemble=127.0.0.1:64316
2024-12-10T15:34:07,293 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,295 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448150x0, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:34:07,303 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44815-0x100105cd0c40001 connected
2024-12-10T15:34:07,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:07,303 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:34:07,303 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:34:07,304 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:34:07,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:34:07,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44815
2024-12-10T15:34:07,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44815
2024-12-10T15:34:07,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44815
2024-12-10T15:34:07,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44815
2024-12-10T15:34:07,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44815
2024-12-10T15:34:07,318 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:35351
2024-12-10T15:34:07,318 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:07,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:07,333 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:34:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,345 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:34:07,346 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,35351,1733844847117 from backup master directory
2024-12-10T15:34:07,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:07,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:07,355 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:34:07,355 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,363 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/hbase.id] with ID: 5db7fd65-7923-421f-83a7-f1555ac02161
2024-12-10T15:34:07,363 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/.tmp/hbase.id
2024-12-10T15:34:07,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:34:07,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:34:07,373 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/.tmp/hbase.id]:[hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/hbase.id]
2024-12-10T15:34:07,388 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:07,388 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:34:07,389 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T15:34:07,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:34:07,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:34:07,402 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:34:07,403 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:34:07,403 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:07,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:34:07,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:34:07,411 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store
2024-12-10T15:34:07,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:34:07,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:34:07,418 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:07,418 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:07,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844847418Disabling compacts and flushes for region at 1733844847418Disabling writes for close at 1733844847418Writing region close event to WAL at 1733844847418Closed at 1733844847418
2024-12-10T15:34:07,419 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/.initializing
2024-12-10T15:34:07,419 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/WALs/d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,422 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C35351%2C1733844847117, suffix=, logDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/WALs/d6bcd7308055,35351,1733844847117, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/oldWALs, maxLogs=10
2024-12-10T15:34:07,422 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C35351%2C1733844847117.1733844847422
2024-12-10T15:34:07,427 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/WALs/d6bcd7308055,35351,1733844847117/d6bcd7308055%2C35351%2C1733844847117.1733844847422
2024-12-10T15:34:07,428 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42873:42873),(127.0.0.1/127.0.0.1:39849:39849)]
2024-12-10T15:34:07,429 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:34:07,429 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:07,429 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,429 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,431 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:07,432 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:34:07,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,433 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:07,433 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:07,435 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:34:07,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,435 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:07,435 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:07,436 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:34:07,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,437 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:07,437 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:07,439 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:34:07,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,439 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:07,440 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,441 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,441 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,444 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,444 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,444 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:34:07,446 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:07,449 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:34:07,449 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785636, jitterRate=-0.0010129362344741821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:34:07,450 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844847430Initializing all the Stores at 1733844847430Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844847430Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844847431 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844847431Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844847431Cleaning up temporary data from old regions at 1733844847444 (+13 ms)Region opened successfully at 1733844847450 (+6 ms)
2024-12-10T15:34:07,451 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:34:07,454 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9ecbba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:34:07,456 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:34:07,456 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:34:07,456 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:34:07,456 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:34:07,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:34:07,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:34:07,457 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:34:07,460 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:34:07,461 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:34:07,470 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:34:07,471 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:34:07,472 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:34:07,481 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:34:07,482 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:34:07,483 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:34:07,492 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:34:07,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:34:07,502 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:34:07,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:07,507 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:34:07,513 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:34:07,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:07,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:07,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,523 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,35351,1733844847117, sessionid=0x100105cd0c40000, setting cluster-up flag (Was=false)
2024-12-10T15:34:07,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,576 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:34:07,580 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:07,628 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:34:07,630 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,35351,1733844847117
2024-12-10T15:34:07,631 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:34:07,632 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:07,632 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:34:07,632 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:34:07,633 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,35351,1733844847117
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:34:07,634 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844877636
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:34:07,636 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:34:07,636 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,636 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:34:07,637 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:34:07,637 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:34:07,637 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,637 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:34:07,637 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:34:07,637 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:34:07,640 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844847637,5,FailOnTimeoutGroup]
2024-12-10T15:34:07,640 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844847640,5,FailOnTimeoutGroup]
2024-12-10T15:34:07,640 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,640 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:34:07,641 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,641 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:34:07,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:34:07,645 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:34:07,645 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3
2024-12-10T15:34:07,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:34:07,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:34:07,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:07,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:34:07,654 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:34:07,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,654 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:07,654 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:34:07,655 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:34:07,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,656 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:07,656 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:34:07,657 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:34:07,657 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,657 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:07,657 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:34:07,658 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:34:07,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:07,659 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:07,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:34:07,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740
2024-12-10T15:34:07,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740
2024-12-10T15:34:07,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:34:07,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:34:07,661 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:34:07,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:34:07,664 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:34:07,664 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852921, jitterRate=0.08454594016075134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844847652Initializing all the Stores at 1733844847653 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844847653Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844847653Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844847653Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844847653Cleaning up temporary data from old regions at 1733844847661 (+8 ms)Region opened successfully at 1733844847665 (+4 ms)
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:34:07,665 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:34:07,665 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:07,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844847665Disabling compacts and flushes for region at 1733844847665Disabling writes for close at 1733844847665Writing region close event to WAL at 1733844847665Closed at 1733844847665
2024-12-10T15:34:07,667 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:07,667 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:34:07,667 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:34:07,668 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:34:07,669 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:34:07,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:07,709 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(746): ClusterId : 5db7fd65-7923-421f-83a7-f1555ac02161
2024-12-10T15:34:07,709 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:34:07,724 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:34:07,724 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:34:07,736 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:34:07,737 DEBUG [RS:0;d6bcd7308055:44815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a29ddf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:34:07,752 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:44815
2024-12-10T15:34:07,752 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:34:07,752 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:34:07,752 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:34:07,752 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,35351,1733844847117 with port=44815, startcode=1733844847291
2024-12-10T15:34:07,753 DEBUG [RS:0;d6bcd7308055:44815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:34:07,754 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:34:07,755 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,755 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,756 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3
2024-12-10T15:34:07,756 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40063
2024-12-10T15:34:07,756 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:34:07,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:34:07,766 DEBUG [RS:0;d6bcd7308055:44815 {}] zookeeper.ZKUtil(111): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,766 WARN  [RS:0;d6bcd7308055:44815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:34:07,766 INFO  [RS:0;d6bcd7308055:44815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:07,766 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,766 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,44815,1733844847291]
2024-12-10T15:34:07,769 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:34:07,772 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:34:07,772 INFO  [RS:0;d6bcd7308055:44815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:34:07,772 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,773 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:34:07,773 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:34:07,774 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:34:07,774 DEBUG [RS:0;d6bcd7308055:44815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,775 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44815,1733844847291-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:34:07,791 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:34:07,791 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44815,1733844847291-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,791 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,791 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.Replication(171): d6bcd7308055,44815,1733844847291 started
2024-12-10T15:34:07,802 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:07,802 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,44815,1733844847291, RpcServer on d6bcd7308055/172.17.0.2:44815, sessionid=0x100105cd0c40001
2024-12-10T15:34:07,802 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:34:07,802 DEBUG [RS:0;d6bcd7308055:44815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,802 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44815,1733844847291'
2024-12-10T15:34:07,802 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,44815,1733844847291
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44815,1733844847291'
2024-12-10T15:34:07,803 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:34:07,804 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:34:07,804 DEBUG [RS:0;d6bcd7308055:44815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:34:07,804 INFO  [RS:0;d6bcd7308055:44815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:34:07,804 INFO  [RS:0;d6bcd7308055:44815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:34:07,819 WARN  [d6bcd7308055:35351 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:34:07,907 INFO  [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44815%2C1733844847291, suffix=, logDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs, maxLogs=32
2024-12-10T15:34:07,908 INFO  [RS:0;d6bcd7308055:44815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44815%2C1733844847291.1733844847908
2024-12-10T15:34:07,917 INFO  [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844847908
2024-12-10T15:34:07,919 DEBUG [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39849:39849),(127.0.0.1/127.0.0.1:42873:42873)]
2024-12-10T15:34:08,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:08,070 DEBUG [d6bcd7308055:35351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:34:08,071 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:08,073 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44815,1733844847291, state=OPENING
2024-12-10T15:34:08,081 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:34:08,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:08,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:08,174 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:34:08,175 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44815,1733844847291}]
2024-12-10T15:34:08,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:08,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:08,331 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:34:08,336 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41759, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:34:08,341 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:34:08,341 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:08,343 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44815%2C1733844847291.meta, suffix=.meta, logDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291, archiveDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs, maxLogs=32
2024-12-10T15:34:08,344 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44815%2C1733844847291.meta.1733844848343.meta
2024-12-10T15:34:08,349 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.meta.1733844848343.meta
2024-12-10T15:34:08,351 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39849:39849),(127.0.0.1/127.0.0.1:42873:42873)]
2024-12-10T15:34:08,352 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:34:08,352 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:34:08,352 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:34:08,352 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:34:08,352 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:34:08,352 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:08,353 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:34:08,353 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:34:08,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:34:08,355 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:34:08,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,356 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:08,356 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:34:08,357 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:34:08,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:08,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:34:08,358 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:34:08,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,358 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:08,358 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:34:08,359 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:34:08,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:08,360 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:34:08,360 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740
2024-12-10T15:34:08,361 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740
2024-12-10T15:34:08,362 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:34:08,362 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:34:08,363 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:34:08,364 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:34:08,364 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741409, jitterRate=-0.057250142097473145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:34:08,364 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:34:08,365 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844848353Writing region info on filesystem at 1733844848353Initializing all the Stores at 1733844848354 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844848354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844848354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844848354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844848354Cleaning up temporary data from old regions at 1733844848362 (+8 ms)Running coprocessor post-open hooks at 1733844848364 (+2 ms)Region opened successfully at 1733844848365 (+1 ms)
2024-12-10T15:34:08,366 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844848330
2024-12-10T15:34:08,368 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:34:08,368 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:34:08,369 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:08,370 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44815,1733844847291, state=OPEN
2024-12-10T15:34:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:34:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:34:08,470 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:08,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:08,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:08,476 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:34:08,477 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44815,1733844847291 in 296 msec
2024-12-10T15:34:08,480 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:34:08,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 810 msec
2024-12-10T15:34:08,482 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:08,482 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:34:08,484 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:34:08,484 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44815,1733844847291, seqNum=-1]
2024-12-10T15:34:08,484 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:34:08,486 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:34:08,493 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 859 msec
2024-12-10T15:34:08,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844848493, completionTime=-1
2024-12-10T15:34:08,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:34:08,493 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:34:08,495 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:34:08,495 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844908495
2024-12-10T15:34:08,495 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733844968495
2024-12-10T15:34:08,495 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:35351, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,496 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,499 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:34:08,500 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.145sec
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:34:08,501 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:34:08,504 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:34:08,504 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:34:08,504 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,35351,1733844847117-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:08,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:08,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac1fb8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:08,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,35351,-1 for getting cluster id
2024-12-10T15:34:08,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:34:08,510 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5db7fd65-7923-421f-83a7-f1555ac02161'
2024-12-10T15:34:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:34:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5db7fd65-7923-421f-83a7-f1555ac02161"

2024-12-10T15:34:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a711054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,35351,-1]
2024-12-10T15:34:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:34:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:08,513 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:34:08,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a524f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:08,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:34:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44815,1733844847291, seqNum=-1]
2024-12-10T15:34:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:34:08,517 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:34:08,519 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,35351,1733844847117
2024-12-10T15:34:08,519 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:08,522 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:34:08,522 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T15:34:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d6bcd7308055,35351,1733844847117
2024-12-10T15:34:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e521a04
2024-12-10T15:34:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T15:34:08,525 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T15:34:08,525 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T15:34:08,525 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T15:34:08,526 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:34:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-10T15:34:08,529 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T15:34:08,529 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,529 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-10T15:34:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:34:08,531 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T15:34:08,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741835_1011 (size=381)
2024-12-10T15:34:08,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741835_1011 (size=381)
2024-12-10T15:34:08,541 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb2b710463a06c67c23fb6d8011bdfd0, NAME => 'TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3
2024-12-10T15:34:08,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741836_1012 (size=64)
2024-12-10T15:34:08,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741836_1012 (size=64)
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing fb2b710463a06c67c23fb6d8011bdfd0, disabling compactions & flushes
2024-12-10T15:34:08,548 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. after waiting 0 ms
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,548 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,548 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb2b710463a06c67c23fb6d8011bdfd0:
Waiting for close lock at 1733844848548Disabling compacts and flushes for region at 1733844848548Disabling writes for close at 1733844848548Writing region close event to WAL at 1733844848548Closed at 1733844848548
2024-12-10T15:34:08,549 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T15:34:08,550 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733844848549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844848549"}]},"ts":"1733844848549"}
2024-12-10T15:34:08,551 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T15:34:08,552 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T15:34:08,553 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844848552"}]},"ts":"1733844848552"}
2024-12-10T15:34:08,554 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-10T15:34:08,555 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, ASSIGN}]
2024-12-10T15:34:08,556 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, ASSIGN
2024-12-10T15:34:08,557 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, ASSIGN; state=OFFLINE, location=d6bcd7308055,44815,1733844847291; forceNewPlan=false, retain=false
2024-12-10T15:34:08,684 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:08,708 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb2b710463a06c67c23fb6d8011bdfd0, regionState=OPENING, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:08,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, ASSIGN because future has completed
2024-12-10T15:34:08,716 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291}]
2024-12-10T15:34:08,884 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,884 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fb2b710463a06c67c23fb6d8011bdfd0, NAME => 'TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:34:08,885 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,885 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:08,885 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,885 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,886 INFO  [StoreOpener-fb2b710463a06c67c23fb6d8011bdfd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fb2b710463a06c67c23fb6d8011bdfd0 
2024-12-10T15:34:08,888 INFO  [StoreOpener-fb2b710463a06c67c23fb6d8011bdfd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb2b710463a06c67c23fb6d8011bdfd0 columnFamilyName info
2024-12-10T15:34:08,888 DEBUG [StoreOpener-fb2b710463a06c67c23fb6d8011bdfd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:08,888 INFO  [StoreOpener-fb2b710463a06c67c23fb6d8011bdfd0-1 {}] regionserver.HStore(327): Store=fb2b710463a06c67c23fb6d8011bdfd0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:08,888 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,889 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,890 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,890 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,890 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,892 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,895 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:34:08,895 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fb2b710463a06c67c23fb6d8011bdfd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=778737, jitterRate=-0.009785071015357971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:34:08,895 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:08,896 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fb2b710463a06c67c23fb6d8011bdfd0:
Running coprocessor pre-open hook at 1733844848885Writing region info on filesystem at 1733844848885Initializing all the Stores at 1733844848886 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844848886Cleaning up temporary data from old regions at 1733844848890 (+4 ms)Running coprocessor post-open hooks at 1733844848895 (+5 ms)Region opened successfully at 1733844848896 (+1 ms)
2024-12-10T15:34:08,897 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., pid=6, masterSystemTime=1733844848874
2024-12-10T15:34:08,899 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,899 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:08,900 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb2b710463a06c67c23fb6d8011bdfd0, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:08,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291 because future has completed
2024-12-10T15:34:08,905 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T15:34:08,905 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291 in 187 msec
2024-12-10T15:34:08,907 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T15:34:08,907 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, ASSIGN in 350 msec
2024-12-10T15:34:08,908 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T15:34:08,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733844848908"}]},"ts":"1733844848908"}
2024-12-10T15:34:08,910 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-10T15:34:08,911 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T15:34:08,913 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 385 msec
2024-12-10T15:34:09,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:09,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,268 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,268 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:09,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:09,796 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:34:09,798 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,799 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,799 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,801 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,822 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,822 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,822 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:09,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:10,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:10,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:10,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:11,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:11,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:11,686 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:11,987 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-10T15:34:11,987 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T15:34:11,988 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T15:34:12,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:12,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:12,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:13,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:13,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:13,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:13,770 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T15:34:13,771 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-10T15:34:14,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:14,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:14,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:15,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:15,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:15,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:16,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:16,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:16,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:17,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:17,495 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:34:17,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,500 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,501 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,503 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,503 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:17,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,525 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,526 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,526 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,529 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:17,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:18,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:18,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T15:34:18,542 INFO  [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-10T15:34:18,542 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-10T15:34:18,549 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-10T15:34:18,549 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., hostname=d6bcd7308055,44815,1733844847291, seqNum=2]
2024-12-10T15:34:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:18,565 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:34:18,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/fa2fe9ff863f4964bedb42f4aff6324e is 1080, key is row0001/info:/1733844858554/Put/seqid=0
2024-12-10T15:34:18,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741837_1013 (size=12509)
2024-12-10T15:34:18,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741837_1013 (size=12509)
2024-12-10T15:34:18,588 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/fa2fe9ff863f4964bedb42f4aff6324e
2024-12-10T15:34:18,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/fa2fe9ff863f4964bedb42f4aff6324e as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e
2024-12-10T15:34:18,600 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e, entries=7, sequenceid=11, filesize=12.2 K
2024-12-10T15:34:18,601 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for fb2b710463a06c67c23fb6d8011bdfd0 in 36ms, sequenceid=11, compaction requested=false
2024-12-10T15:34:18,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:18,602 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-10T15:34:18,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/54fd51b84b1b4c0da71efaa99b5dc3d4 is 1080, key is row0008/info:/1733844858566/Put/seqid=0
2024-12-10T15:34:18,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741838_1014 (size=24376)
2024-12-10T15:34:18,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741838_1014 (size=24376)
2024-12-10T15:34:18,611 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/54fd51b84b1b4c0da71efaa99b5dc3d4
2024-12-10T15:34:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/54fd51b84b1b4c0da71efaa99b5dc3d4 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4
2024-12-10T15:34:18,621 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4, entries=18, sequenceid=32, filesize=23.8 K
2024-12-10T15:34:18,622 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=7.36 KB/7532 for fb2b710463a06c67c23fb6d8011bdfd0 in 20ms, sequenceid=32, compaction requested=false
2024-12-10T15:34:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.0 K, sizeToCheck=16.0 K
2024-12-10T15:34:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4 because midkey is the same as first or last row
2024-12-10T15:34:18,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:19,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:19,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:19,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:20,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:20,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:20,621 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-10T15:34:20,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/2c518d9217a14ae4b202ad9e5772724c is 1080, key is row0026/info:/1733844858603/Put/seqid=0
2024-12-10T15:34:20,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741839_1015 (size=13586)
2024-12-10T15:34:20,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741839_1015 (size=13586)
2024-12-10T15:34:20,634 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/2c518d9217a14ae4b202ad9e5772724c
2024-12-10T15:34:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/2c518d9217a14ae4b202ad9e5772724c as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c
2024-12-10T15:34:20,646 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c, entries=8, sequenceid=43, filesize=13.3 K
2024-12-10T15:34:20,647 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=10.51 KB/10760 for fb2b710463a06c67c23fb6d8011bdfd0 in 26ms, sequenceid=43, compaction requested=true
2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4 because midkey is the same as first or last row
2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2b710463a06c67c23fb6d8011bdfd0:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:20,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:20,647 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:20,648 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:20,648 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): fb2b710463a06c67c23fb6d8011bdfd0/info is initiating minor compaction (all files)
2024-12-10T15:34:20,649 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2b710463a06c67c23fb6d8011bdfd0/info in TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:20,649 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:20,649 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp, totalSize=49.3 K
2024-12-10T15:34:20,649 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2fe9ff863f4964bedb42f4aff6324e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733844858554
2024-12-10T15:34:20,650 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54fd51b84b1b4c0da71efaa99b5dc3d4, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733844858566
2024-12-10T15:34:20,650 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c518d9217a14ae4b202ad9e5772724c, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733844858603
2024-12-10T15:34:20,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/cc243e4abf2a40eb9cb638f4acc64271 is 1080, key is row0034/info:/1733844860622/Put/seqid=0
2024-12-10T15:34:20,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741840_1016 (size=17894)
2024-12-10T15:34:20,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741840_1016 (size=17894)
2024-12-10T15:34:20,659 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/cc243e4abf2a40eb9cb638f4acc64271
2024-12-10T15:34:20,665 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2b710463a06c67c23fb6d8011bdfd0#info#compaction#60 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:20,666 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/3524454dd13c47488bd7b46a2bcfe4c5 is 1080, key is row0001/info:/1733844858554/Put/seqid=0
2024-12-10T15:34:20,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/cc243e4abf2a40eb9cb638f4acc64271 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271
2024-12-10T15:34:20,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741841_1017 (size=40670)
2024-12-10T15:34:20,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741841_1017 (size=40670)
2024-12-10T15:34:20,672 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271, entries=12, sequenceid=58, filesize=17.5 K
2024-12-10T15:34:20,673 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for fb2b710463a06c67c23fb6d8011bdfd0 in 24ms, sequenceid=58, compaction requested=false
2024-12-10T15:34:20,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:20,673 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.8 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,673 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,673 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4 because midkey is the same as first or last row
2024-12-10T15:34:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:20,674 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-10T15:34:20,678 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/3524454dd13c47488bd7b46a2bcfe4c5 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5
2024-12-10T15:34:20,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e6122679897c4b01a665b93cb6d6db96 is 1080, key is row0046/info:/1733844860650/Put/seqid=0
2024-12-10T15:34:20,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741842_1018 (size=16817)
2024-12-10T15:34:20,685 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2b710463a06c67c23fb6d8011bdfd0/info of fb2b710463a06c67c23fb6d8011bdfd0 into 3524454dd13c47488bd7b46a2bcfe4c5(size=39.7 K), total size for store is 57.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:20,685 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:20,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741842_1018 (size=16817)
2024-12-10T15:34:20,685 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., storeName=fb2b710463a06c67c23fb6d8011bdfd0/info, priority=13, startTime=1733844860647; duration=0sec
2024-12-10T15:34:20,685 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,685 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,685 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5 because midkey is the same as first or last row
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5 because midkey is the same as first or last row
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5 because midkey is the same as first or last row
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:20,686 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2b710463a06c67c23fb6d8011bdfd0:info
2024-12-10T15:34:20,686 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e6122679897c4b01a665b93cb6d6db96
2024-12-10T15:34:20,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e6122679897c4b01a665b93cb6d6db96 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96
2024-12-10T15:34:20,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:20,698 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96, entries=11, sequenceid=72, filesize=16.4 K
2024-12-10T15:34:20,698 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for fb2b710463a06c67c23fb6d8011bdfd0 in 24ms, sequenceid=72, compaction requested=true
2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.6 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5 because midkey is the same as first or last row
2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2b710463a06c67c23fb6d8011bdfd0:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:20,699 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:20,700 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 75381 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:20,700 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): fb2b710463a06c67c23fb6d8011bdfd0/info is initiating minor compaction (all files)
2024-12-10T15:34:20,700 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2b710463a06c67c23fb6d8011bdfd0/info in TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:20,700 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp, totalSize=73.6 K
2024-12-10T15:34:20,701 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3524454dd13c47488bd7b46a2bcfe4c5, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733844858554
2024-12-10T15:34:20,701 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc243e4abf2a40eb9cb638f4acc64271, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733844860622
2024-12-10T15:34:20,701 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6122679897c4b01a665b93cb6d6db96, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1733844860650
2024-12-10T15:34:20,713 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2b710463a06c67c23fb6d8011bdfd0#info#compaction#62 average throughput is 28.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:20,713 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/64c7bd3eb0b94355824b4d87006161bf is 1080, key is row0001/info:/1733844858554/Put/seqid=0
2024-12-10T15:34:20,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741843_1019 (size=65612)
2024-12-10T15:34:20,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741843_1019 (size=65612)
2024-12-10T15:34:20,728 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/64c7bd3eb0b94355824b4d87006161bf as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf
2024-12-10T15:34:20,736 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2b710463a06c67c23fb6d8011bdfd0/info of fb2b710463a06c67c23fb6d8011bdfd0 into 64c7bd3eb0b94355824b4d87006161bf(size=64.1 K), total size for store is 64.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:20,736 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., storeName=fb2b710463a06c67c23fb6d8011bdfd0/info, priority=13, startTime=1733844860699; duration=0sec
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf because midkey is the same as first or last row
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf because midkey is the same as first or last row
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf because midkey is the same as first or last row
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:20,736 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2b710463a06c67c23fb6d8011bdfd0:info
2024-12-10T15:34:21,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:21,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:21,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:22,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:22,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:22,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:22,699 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-10T15:34:22,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/571379d4c1f8417ebb7b720d4e194cae is 1080, key is row0057/info:/1733844860676/Put/seqid=0
2024-12-10T15:34:22,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741844_1020 (size=14663)
2024-12-10T15:34:22,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741844_1020 (size=14663)
2024-12-10T15:34:22,714 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/571379d4c1f8417ebb7b720d4e194cae
2024-12-10T15:34:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/571379d4c1f8417ebb7b720d4e194cae as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae
2024-12-10T15:34:22,727 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae, entries=9, sequenceid=86, filesize=14.3 K
2024-12-10T15:34:22,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=10.51 KB/10760 for fb2b710463a06c67c23fb6d8011bdfd0 in 30ms, sequenceid=86, compaction requested=false
2024-12-10T15:34:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf because midkey is the same as first or last row
2024-12-10T15:34:22,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-10T15:34:22,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e3ca686d523f43868bff362ffcdf7ec4 is 1080, key is row0066/info:/1733844862702/Put/seqid=0
2024-12-10T15:34:22,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741845_1021 (size=16817)
2024-12-10T15:34:22,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741845_1021 (size=16817)
2024-12-10T15:34:22,742 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e3ca686d523f43868bff362ffcdf7ec4
2024-12-10T15:34:22,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/e3ca686d523f43868bff362ffcdf7ec4 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4
2024-12-10T15:34:22,753 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4, entries=11, sequenceid=100, filesize=16.4 K
2024-12-10T15:34:22,754 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for fb2b710463a06c67c23fb6d8011bdfd0 in 26ms, sequenceid=100, compaction requested=true
2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=94.8 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf because midkey is the same as first or last row
2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2b710463a06c67c23fb6d8011bdfd0:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:22,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:22,754 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:22,755 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T15:34:22,755 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 97092 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:22,756 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): fb2b710463a06c67c23fb6d8011bdfd0/info is initiating minor compaction (all files)
2024-12-10T15:34:22,756 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2b710463a06c67c23fb6d8011bdfd0/info in TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:22,756 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp, totalSize=94.8 K
2024-12-10T15:34:22,756 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c7bd3eb0b94355824b4d87006161bf, keycount=56, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1733844858554
2024-12-10T15:34:22,757 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571379d4c1f8417ebb7b720d4e194cae, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733844860676
2024-12-10T15:34:22,757 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ca686d523f43868bff362ffcdf7ec4, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733844862702
2024-12-10T15:34:22,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/96b428454a1c42c8b1857d084bd395ea is 1080, key is row0077/info:/1733844862729/Put/seqid=0
2024-12-10T15:34:22,770 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2b710463a06c67c23fb6d8011bdfd0#info#compaction#66 average throughput is 38.99 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:22,771 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/cfbc9fd651f34d62a0e9c8398284f285 is 1080, key is row0001/info:/1733844858554/Put/seqid=0
2024-12-10T15:34:22,787 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741846_1022 (size=20064)
2024-12-10T15:34:22,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741846_1022 (size=20064)
2024-12-10T15:34:22,788 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/96b428454a1c42c8b1857d084bd395ea
2024-12-10T15:34:22,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741847_1023 (size=87327)
2024-12-10T15:34:22,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741847_1023 (size=87327)
2024-12-10T15:34:22,794 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/cfbc9fd651f34d62a0e9c8398284f285 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285
2024-12-10T15:34:22,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/96b428454a1c42c8b1857d084bd395ea as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/96b428454a1c42c8b1857d084bd395ea
2024-12-10T15:34:22,800 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/96b428454a1c42c8b1857d084bd395ea, entries=14, sequenceid=117, filesize=19.6 K
2024-12-10T15:34:22,801 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=6.30 KB/6456 for fb2b710463a06c67c23fb6d8011bdfd0 in 46ms, sequenceid=117, compaction requested=false
2024-12-10T15:34:22,801 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2b710463a06c67c23fb6d8011bdfd0/info of fb2b710463a06c67c23fb6d8011bdfd0 into cfbc9fd651f34d62a0e9c8398284f285(size=85.3 K), total size for store is 104.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:22,801 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2b710463a06c67c23fb6d8011bdfd0:

2024-12-10T15:34:22,801 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., storeName=fb2b710463a06c67c23fb6d8011bdfd0/info, priority=13, startTime=1733844862754; duration=0sec
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,801 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,801 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:22,802 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,802 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,802 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-12-10T15:34:22,802 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T15:34:22,803 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1
2024-12-10T15:34:22,803 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1
2024-12-10T15:34:22,803 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2b710463a06c67c23fb6d8011bdfd0:info
2024-12-10T15:34:22,803 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] assignment.AssignmentManager(1363): Split request from d6bcd7308055,44815,1733844847291, parent={ENCODED => fb2b710463a06c67c23fb6d8011bdfd0, NAME => 'TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-10T15:34:22,808 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:22,811 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=3c8e97fa70d30e286e3c3552dfd2be3f, daughterB=84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:22,812 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=3c8e97fa70d30e286e3c3552dfd2be3f, daughterB=84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:22,812 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=3c8e97fa70d30e286e3c3552dfd2be3f, daughterB=84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:22,812 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=3c8e97fa70d30e286e3c3552dfd2be3f, daughterB=84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:22,813 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] assignment.AssignmentManager(1363): Split request from d6bcd7308055,44815,1733844847291, parent={ENCODED => fb2b710463a06c67c23fb6d8011bdfd0, NAME => 'TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-10T15:34:22,814 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=SPLITTING, location=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:22,815 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35351 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06
2024-12-10T15:34:22,815 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06 held by pid=7
2024-12-10T15:34:22,818 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, UNASSIGN}]
2024-12-10T15:34:22,822 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(162): LOCK_EVENT_WAIT SchemaLocking[serverLocks={},namespaceLocks={hbase=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},tableLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},regionLocks={},peerLocks={},metaLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},globalLocks={}]
2024-12-10T15:34:22,822 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, UNASSIGN
2024-12-10T15:34:22,822 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06
2024-12-10T15:34:22,824 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fb2b710463a06c67c23fb6d8011bdfd0, regionState=CLOSING, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:22,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, UNASSIGN because future has completed
2024-12-10T15:34:22,826 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-10T15:34:22,826 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291}]
2024-12-10T15:34:22,983 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:22,983 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-10T15:34:22,983 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing fb2b710463a06c67c23fb6d8011bdfd0, disabling compactions & flushes
2024-12-10T15:34:22,983 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:22,983 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:22,983 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. after waiting 0 ms
2024-12-10T15:34:22,983 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:22,983 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(2902): Flushing fb2b710463a06c67c23fb6d8011bdfd0 1/1 column families, dataSize=6.30 KB heapSize=7 KB
2024-12-10T15:34:22,987 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/f65bdd695fd5452cbf5d2522cdc4dab7 is 1080, key is row0091/info:/1733844862756/Put/seqid=0
2024-12-10T15:34:22,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741848_1024 (size=11424)
2024-12-10T15:34:22,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741848_1024 (size=11424)
2024-12-10T15:34:22,993 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/f65bdd695fd5452cbf5d2522cdc4dab7
2024-12-10T15:34:22,998 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/.tmp/info/f65bdd695fd5452cbf5d2522cdc4dab7 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/f65bdd695fd5452cbf5d2522cdc4dab7
2024-12-10T15:34:23,003 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/f65bdd695fd5452cbf5d2522cdc4dab7, entries=6, sequenceid=127, filesize=11.2 K
2024-12-10T15:34:23,004 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for fb2b710463a06c67c23fb6d8011bdfd0 in 21ms, sequenceid=127, compaction requested=true
2024-12-10T15:34:23,005 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4] to archive
2024-12-10T15:34:23,006 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:34:23,009 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/fa2fe9ff863f4964bedb42f4aff6324e
2024-12-10T15:34:23,009 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/54fd51b84b1b4c0da71efaa99b5dc3d4
2024-12-10T15:34:23,009 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/3524454dd13c47488bd7b46a2bcfe4c5
2024-12-10T15:34:23,009 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/2c518d9217a14ae4b202ad9e5772724c
2024-12-10T15:34:23,013 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e6122679897c4b01a665b93cb6d6db96
2024-12-10T15:34:23,013 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/e3ca686d523f43868bff362ffcdf7ec4
2024-12-10T15:34:23,013 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/571379d4c1f8417ebb7b720d4e194cae
2024-12-10T15:34:23,013 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/64c7bd3eb0b94355824b4d87006161bf
2024-12-10T15:34:23,014 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cc243e4abf2a40eb9cb638f4acc64271
2024-12-10T15:34:23,020 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-12-10T15:34:23,020 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.
2024-12-10T15:34:23,021 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for fb2b710463a06c67c23fb6d8011bdfd0:
Waiting for close lock at 1733844862983Running coprocessor pre-close hooks at 1733844862983Disabling compacts and flushes for region at 1733844862983Disabling writes for close at 1733844862983Obtaining lock to block concurrent updates at 1733844862983Preparing flush snapshotting stores in fb2b710463a06c67c23fb6d8011bdfd0 at 1733844862983Finished memstore snapshotting TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., syncing WAL and waiting on mvcc, flushsize=dataSize=6456, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1733844862984 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. at 1733844862984Flushing fb2b710463a06c67c23fb6d8011bdfd0/info: creating writer at 1733844862984Flushing fb2b710463a06c67c23fb6d8011bdfd0/info: appending metadata at 1733844862987 (+3 ms)Flushing fb2b710463a06c67c23fb6d8011bdfd0/info: closing flushed file at 1733844862987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14253f4c: reopening flushed file at 1733844862997 (+10 ms)Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for fb2b710463a06c67c23fb6d8011bdfd0 in 21ms, sequenceid=127, compaction requested=true at 1733844863004 (+7 ms)Writing region close event to WAL at 1733844863016 (+12 ms)Running coprocessor post-close hooks at 1733844863020 (+4 ms)Closed at 1733844863020
2024-12-10T15:34:23,023 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,023 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fb2b710463a06c67c23fb6d8011bdfd0, regionState=CLOSED
2024-12-10T15:34:23,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291 because future has completed
2024-12-10T15:34:23,028 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-10T15:34:23,028 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure fb2b710463a06c67c23fb6d8011bdfd0, server=d6bcd7308055,44815,1733844847291 in 200 msec
2024-12-10T15:34:23,030 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7
2024-12-10T15:34:23,030 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fb2b710463a06c67c23fb6d8011bdfd0, UNASSIGN in 210 msec
2024-12-10T15:34:23,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:23,037 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:23,040 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=fb2b710463a06c67c23fb6d8011bdfd0, threads=3
2024-12-10T15:34:23,042 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/f65bdd695fd5452cbf5d2522cdc4dab7 for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,042 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285 for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,042 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/96b428454a1c42c8b1857d084bd395ea for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,050 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/f65bdd695fd5452cbf5d2522cdc4dab7, top=true
2024-12-10T15:34:23,050 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/96b428454a1c42c8b1857d084bd395ea, top=true
2024-12-10T15:34:23,062 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7 for child: 84446645ea9a814c7eb236a96a7609ff, parent: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,062 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea for child: 84446645ea9a814c7eb236a96a7609ff, parent: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,062 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/f65bdd695fd5452cbf5d2522cdc4dab7 for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741849_1025 (size=27)
2024-12-10T15:34:23,062 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/96b428454a1c42c8b1857d084bd395ea for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741849_1025 (size=27)
2024-12-10T15:34:23,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741850_1026 (size=27)
2024-12-10T15:34:23,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741850_1026 (size=27)
2024-12-10T15:34:23,073 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285 for region: fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:23,074 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region fb2b710463a06c67c23fb6d8011bdfd0 Daughter A: [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0] storefiles, Daughter B: [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7] storefiles.
2024-12-10T15:34:23,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741851_1027 (size=71)
2024-12-10T15:34:23,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741851_1027 (size=71)
2024-12-10T15:34:23,083 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:23,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741852_1028 (size=71)
2024-12-10T15:34:23,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741852_1028 (size=71)
2024-12-10T15:34:23,100 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:23,109 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-10T15:34:23,110 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-10T15:34:23,112 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733844863112"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733844863112"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733844863112"}]},"ts":"1733844863112"}
2024-12-10T15:34:23,113 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733844863112"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844863112"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733844863112"}]},"ts":"1733844863112"}
2024-12-10T15:34:23,113 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733844863112"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733844863112"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733844863112"}]},"ts":"1733844863112"}
2024-12-10T15:34:23,127 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3c8e97fa70d30e286e3c3552dfd2be3f, ASSIGN}, {pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=84446645ea9a814c7eb236a96a7609ff, ASSIGN}]
2024-12-10T15:34:23,128 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=84446645ea9a814c7eb236a96a7609ff, ASSIGN
2024-12-10T15:34:23,128 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3c8e97fa70d30e286e3c3552dfd2be3f, ASSIGN
2024-12-10T15:34:23,129 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=84446645ea9a814c7eb236a96a7609ff, ASSIGN; state=SPLITTING_NEW, location=d6bcd7308055,44815,1733844847291; forceNewPlan=false, retain=false
2024-12-10T15:34:23,129 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3c8e97fa70d30e286e3c3552dfd2be3f, ASSIGN; state=SPLITTING_NEW, location=d6bcd7308055,44815,1733844847291; forceNewPlan=false, retain=false
2024-12-10T15:34:23,280 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=84446645ea9a814c7eb236a96a7609ff, regionState=OPENING, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:23,280 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3c8e97fa70d30e286e3c3552dfd2be3f, regionState=OPENING, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:23,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3c8e97fa70d30e286e3c3552dfd2be3f, ASSIGN because future has completed
2024-12-10T15:34:23,283 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8e97fa70d30e286e3c3552dfd2be3f, server=d6bcd7308055,44815,1733844847291}]
2024-12-10T15:34:23,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=84446645ea9a814c7eb236a96a7609ff, ASSIGN because future has completed
2024-12-10T15:34:23,285 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84446645ea9a814c7eb236a96a7609ff, server=d6bcd7308055,44815,1733844847291}]
2024-12-10T15:34:23,447 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:23,448 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c8e97fa70d30e286e3c3552dfd2be3f, NAME => 'TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-10T15:34:23,448 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,448 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:23,448 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,448 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,449 INFO  [StoreOpener-3c8e97fa70d30e286e3c3552dfd2be3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3c8e97fa70d30e286e3c3552dfd2be3f 
2024-12-10T15:34:23,450 INFO  [StoreOpener-3c8e97fa70d30e286e3c3552dfd2be3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8e97fa70d30e286e3c3552dfd2be3f columnFamilyName info
2024-12-10T15:34:23,450 DEBUG [StoreOpener-3c8e97fa70d30e286e3c3552dfd2be3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:23,459 DEBUG [StoreOpener-3c8e97fa70d30e286e3c3552dfd2be3f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-bottom
2024-12-10T15:34:23,460 INFO  [StoreOpener-3c8e97fa70d30e286e3c3552dfd2be3f-1 {}] regionserver.HStore(327): Store=3c8e97fa70d30e286e3c3552dfd2be3f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:23,460 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,461 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,462 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,462 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,462 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,464 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,464 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 3c8e97fa70d30e286e3c3552dfd2be3f; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791813, jitterRate=0.0068426430225372314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:34:23,464 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:23,465 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 3c8e97fa70d30e286e3c3552dfd2be3f:
Running coprocessor pre-open hook at 1733844863448Writing region info on filesystem at 1733844863448Initializing all the Stores at 1733844863449 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844863449Cleaning up temporary data from old regions at 1733844863462 (+13 ms)Running coprocessor post-open hooks at 1733844863464 (+2 ms)Region opened successfully at 1733844863465 (+1 ms)
2024-12-10T15:34:23,466 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f., pid=13, masterSystemTime=1733844863436
2024-12-10T15:34:23,466 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 3c8e97fa70d30e286e3c3552dfd2be3f:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:23,466 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:23,466 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-10T15:34:23,466 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:23,466 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 3c8e97fa70d30e286e3c3552dfd2be3f/info is initiating minor compaction (all files)
2024-12-10T15:34:23,466 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c8e97fa70d30e286e3c3552dfd2be3f/info in TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:23,467 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-bottom] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/.tmp, totalSize=85.3 K
2024-12-10T15:34:23,467 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0, keycount=38, bloomtype=ROW, size=85.3 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733844858554
2024-12-10T15:34:23,468 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:23,468 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:23,468 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:23,468 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => 84446645ea9a814c7eb236a96a7609ff, NAME => 'TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-10T15:34:23,468 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,468 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:23,468 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,468 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3c8e97fa70d30e286e3c3552dfd2be3f, regionState=OPEN, openSeqNum=131, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:23,469 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,470 INFO  [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 84446645ea9a814c7eb236a96a7609ff 
2024-12-10T15:34:23,470 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-10T15:34:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-10T15:34:23,471 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-12-10T15:34:23,471 INFO  [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84446645ea9a814c7eb236a96a7609ff columnFamilyName info
2024-12-10T15:34:23,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8e97fa70d30e286e3c3552dfd2be3f, server=d6bcd7308055,44815,1733844847291 because future has completed
2024-12-10T15:34:23,471 DEBUG [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:23,480 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-12-10T15:34:23,480 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c8e97fa70d30e286e3c3552dfd2be3f, server=d6bcd7308055,44815,1733844847291 in 194 msec
2024-12-10T15:34:23,482 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3c8e97fa70d30e286e3c3552dfd2be3f, ASSIGN in 353 msec
2024-12-10T15:34:23,489 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c8e97fa70d30e286e3c3552dfd2be3f#info#compaction#68 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:23,489 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/.tmp/info/d97f4b80fe58494fb630d7021c1f5e3f is 1080, key is row0001/info:/1733844858554/Put/seqid=0
2024-12-10T15:34:23,491 DEBUG [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea
2024-12-10T15:34:23,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741853_1029 (size=70862)
2024-12-10T15:34:23,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741853_1029 (size=70862)
2024-12-10T15:34:23,496 DEBUG [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7
2024-12-10T15:34:23,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/41a783c68dd545b3a3b238155e9be806 is 193, key is TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff./info:regioninfo/1733844863280/Put/seqid=0
2024-12-10T15:34:23,499 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/.tmp/info/d97f4b80fe58494fb630d7021c1f5e3f as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/d97f4b80fe58494fb630d7021c1f5e3f
2024-12-10T15:34:23,500 DEBUG [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-top
2024-12-10T15:34:23,501 INFO  [StoreOpener-84446645ea9a814c7eb236a96a7609ff-1 {}] regionserver.HStore(327): Store=84446645ea9a814c7eb236a96a7609ff/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:23,501 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,502 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741854_1030 (size=9847)
2024-12-10T15:34:23,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741854_1030 (size=9847)
2024-12-10T15:34:23,503 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/41a783c68dd545b3a3b238155e9be806
2024-12-10T15:34:23,503 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,504 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,504 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,505 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,506 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 3c8e97fa70d30e286e3c3552dfd2be3f/info of 3c8e97fa70d30e286e3c3552dfd2be3f into d97f4b80fe58494fb630d7021c1f5e3f(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:23,506 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c8e97fa70d30e286e3c3552dfd2be3f:

2024-12-10T15:34:23,506 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f., storeName=3c8e97fa70d30e286e3c3552dfd2be3f/info, priority=15, startTime=1733844863466; duration=0sec
2024-12-10T15:34:23,506 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened 84446645ea9a814c7eb236a96a7609ff; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781856, jitterRate=-0.005819261074066162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T15:34:23,506 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:23,506 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:23,506 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for 84446645ea9a814c7eb236a96a7609ff:
Running coprocessor pre-open hook at 1733844863469Writing region info on filesystem at 1733844863469Initializing all the Stores at 1733844863469Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844863469Cleaning up temporary data from old regions at 1733844863504 (+35 ms)Running coprocessor post-open hooks at 1733844863506 (+2 ms)Region opened successfully at 1733844863506
2024-12-10T15:34:23,506 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c8e97fa70d30e286e3c3552dfd2be3f:info
2024-12-10T15:34:23,507 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., pid=14, masterSystemTime=1733844863436
2024-12-10T15:34:23,507 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:23,507 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:23,507 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:23,508 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:23,508 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:23,508 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:23,509 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-top, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=116.0 K
2024-12-10T15:34:23,509 DEBUG [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:23,509 INFO  [RS_OPEN_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:23,509 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0, keycount=38, bloomtype=ROW, size=85.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733844858554
2024-12-10T15:34:23,509 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733844862729
2024-12-10T15:34:23,510 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733844862756
2024-12-10T15:34:23,510 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=84446645ea9a814c7eb236a96a7609ff, regionState=OPEN, openSeqNum=131, regionLocation=d6bcd7308055,44815,1733844847291
2024-12-10T15:34:23,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84446645ea9a814c7eb236a96a7609ff, server=d6bcd7308055,44815,1733844847291 because future has completed
2024-12-10T15:34:23,516 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=12
2024-12-10T15:34:23,516 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 84446645ea9a814c7eb236a96a7609ff, server=d6bcd7308055,44815,1733844847291 in 228 msec
2024-12-10T15:34:23,519 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=7
2024-12-10T15:34:23,519 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=84446645ea9a814c7eb236a96a7609ff, ASSIGN in 389 msec
2024-12-10T15:34:23,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:23,521 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=3c8e97fa70d30e286e3c3552dfd2be3f, daughterB=84446645ea9a814c7eb236a96a7609ff in 711 msec
2024-12-10T15:34:23,521 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06
2024-12-10T15:34:23,521 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06
2024-12-10T15:34:23,521 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06
2024-12-10T15:34:23,522 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(534): Split of {ENCODED => fb2b710463a06c67c23fb6d8011bdfd0, NAME => 'TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0.', STARTKEY => '', ENDKEY => ''} skipped; state is already SPLIT
2024-12-10T15:34:23,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/ns/338a63eb3f9643d7ac058eee63ae218e is 43, key is default/ns:d/1733844848486/Put/seqid=0
2024-12-10T15:34:23,523 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=fb2b710463a06c67c23fb6d8011bdfd0, daughterA=df78e1c48a6f00aba9cb0c867323f90a, daughterB=ded92b11dd3673ec2727c2c29349aa06 in 708 msec
2024-12-10T15:34:23,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741855_1031 (size=5153)
2024-12-10T15:34:23,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741855_1031 (size=5153)
2024-12-10T15:34:23,527 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/ns/338a63eb3f9643d7ac058eee63ae218e
2024-12-10T15:34:23,532 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#71 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:23,532 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ecf9714275bd4046acd5f7fa7819337b is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:23,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741856_1032 (size=42984)
2024-12-10T15:34:23,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741856_1032 (size=42984)
2024-12-10T15:34:23,546 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ecf9714275bd4046acd5f7fa7819337b as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ecf9714275bd4046acd5f7fa7819337b
2024-12-10T15:34:23,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/table/9c9751b982f34cc5930b79d881c66b6c is 65, key is TestLogRolling-testLogRolling/table:state/1733844848908/Put/seqid=0
2024-12-10T15:34:23,552 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into ecf9714275bd4046acd5f7fa7819337b(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:23,552 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:23,552 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844863507; duration=0sec
2024-12-10T15:34:23,552 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:23,553 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:23,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741857_1033 (size=5340)
2024-12-10T15:34:23,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741857_1033 (size=5340)
2024-12-10T15:34:23,556 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/table/9c9751b982f34cc5930b79d881c66b6c
2024-12-10T15:34:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/41a783c68dd545b3a3b238155e9be806 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/info/41a783c68dd545b3a3b238155e9be806
2024-12-10T15:34:23,567 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/info/41a783c68dd545b3a3b238155e9be806, entries=30, sequenceid=17, filesize=9.6 K
2024-12-10T15:34:23,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/ns/338a63eb3f9643d7ac058eee63ae218e as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/ns/338a63eb3f9643d7ac058eee63ae218e
2024-12-10T15:34:23,574 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/ns/338a63eb3f9643d7ac058eee63ae218e, entries=2, sequenceid=17, filesize=5.0 K
2024-12-10T15:34:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/table/9c9751b982f34cc5930b79d881c66b6c as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/table/9c9751b982f34cc5930b79d881c66b6c
2024-12-10T15:34:23,580 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/table/9c9751b982f34cc5930b79d881c66b6c, entries=2, sequenceid=17, filesize=5.2 K
2024-12-10T15:34:23,582 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 110ms, sequenceid=17, compaction requested=false
2024-12-10T15:34:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T15:34:23,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:24,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:24,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:24,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:38976 deadline: 1733844874770, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. is not online on d6bcd7308055,44815,1733844847291
2024-12-10T15:34:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., hostname=d6bcd7308055,44815,1733844847291, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., hostname=d6bcd7308055,44815,1733844847291, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. is not online on d6bcd7308055,44815,1733844847291
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T15:34:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., hostname=d6bcd7308055,44815,1733844847291, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0. is not online on d6bcd7308055,44815,1733844847291
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T15:34:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733844848525.fb2b710463a06c67c23fb6d8011bdfd0., hostname=d6bcd7308055,44815,1733844847291, seqNum=2 from cache
2024-12-10T15:34:25,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:25,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:25,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:26,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:26,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:26,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:27,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:27,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:27,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:28,022 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,023 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,024 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,024 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,025 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,025 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:28,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,049 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:28,554 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T15:34:28,555 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,556 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,557 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,557 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,558 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,575 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,575 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,578 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,579 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,579 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,581 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T15:34:28,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:29,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:29,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:29,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:30,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:30,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:30,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:31,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:31,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:31,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:32,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:32,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:32,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:33,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:33,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:33,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:34,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:34,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:34,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., hostname=d6bcd7308055,44815,1733844847291, seqNum=131]
2024-12-10T15:34:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:34,853 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T15:34:34,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1ac96179c7fc4cefb5fc5e8e660d4ed1 is 1080, key is row0097/info:/1733844874844/Put/seqid=0
2024-12-10T15:34:34,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741858_1034 (size=12516)
2024-12-10T15:34:34,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741858_1034 (size=12516)
2024-12-10T15:34:34,864 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1ac96179c7fc4cefb5fc5e8e660d4ed1
2024-12-10T15:34:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1ac96179c7fc4cefb5fc5e8e660d4ed1 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1
2024-12-10T15:34:34,875 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1, entries=7, sequenceid=141, filesize=12.2 K
2024-12-10T15:34:34,876 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 84446645ea9a814c7eb236a96a7609ff in 23ms, sequenceid=141, compaction requested=false
2024-12-10T15:34:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:34,878 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:34,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ac132cbd1f9b4d13a9ff247582e5c7cc is 1080, key is row0104/info:/1733844874854/Put/seqid=0
2024-12-10T15:34:34,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741859_1035 (size=17906)
2024-12-10T15:34:34,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741859_1035 (size=17906)
2024-12-10T15:34:34,891 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ac132cbd1f9b4d13a9ff247582e5c7cc
2024-12-10T15:34:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ac132cbd1f9b4d13a9ff247582e5c7cc as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc
2024-12-10T15:34:34,902 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc, entries=12, sequenceid=156, filesize=17.5 K
2024-12-10T15:34:34,902 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=13.66 KB/13988 for 84446645ea9a814c7eb236a96a7609ff in 24ms, sequenceid=156, compaction requested=true
2024-12-10T15:34:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:34,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:34,903 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:34,903 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 73406 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:34,904 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:34,904 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:34,904 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ecf9714275bd4046acd5f7fa7819337b, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=71.7 K
2024-12-10T15:34:34,904 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf9714275bd4046acd5f7fa7819337b, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733844860688
2024-12-10T15:34:34,904 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac96179c7fc4cefb5fc5e8e660d4ed1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733844874844
2024-12-10T15:34:34,905 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac132cbd1f9b4d13a9ff247582e5c7cc, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733844874854
2024-12-10T15:34:34,913 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#75 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:34,913 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/132db4ff902a459a948567df93c651dd is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:34,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741860_1036 (size=63636)
2024-12-10T15:34:34,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741860_1036 (size=63636)
2024-12-10T15:34:34,923 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/132db4ff902a459a948567df93c651dd as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/132db4ff902a459a948567df93c651dd
2024-12-10T15:34:34,928 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into 132db4ff902a459a948567df93c651dd(size=62.1 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:34,928 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:34,928 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844874902; duration=0sec
2024-12-10T15:34:34,928 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:34,928 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:35,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:35,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:35,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:36,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:36,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:36,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:36,908 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T15:34:36,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1cd6a4719a5946fc861d0e1fed91a97c is 1080, key is row0116/info:/1733844874879/Put/seqid=0
2024-12-10T15:34:36,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741861_1037 (size=20078)
2024-12-10T15:34:36,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741861_1037 (size=20078)
2024-12-10T15:34:36,920 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1cd6a4719a5946fc861d0e1fed91a97c
2024-12-10T15:34:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/1cd6a4719a5946fc861d0e1fed91a97c as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c
2024-12-10T15:34:36,931 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c, entries=14, sequenceid=174, filesize=19.6 K
2024-12-10T15:34:36,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=10.51 KB/10760 for 84446645ea9a814c7eb236a96a7609ff in 24ms, sequenceid=174, compaction requested=false
2024-12-10T15:34:36,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:36,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-10T15:34:36,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/262762f9d0d540fb83224699f485b5d4 is 1080, key is row0130/info:/1733844876911/Put/seqid=0
2024-12-10T15:34:36,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741862_1038 (size=16828)
2024-12-10T15:34:36,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741862_1038 (size=16828)
2024-12-10T15:34:36,941 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/262762f9d0d540fb83224699f485b5d4
2024-12-10T15:34:36,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/262762f9d0d540fb83224699f485b5d4 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4
2024-12-10T15:34:36,953 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4, entries=11, sequenceid=188, filesize=16.4 K
2024-12-10T15:34:36,954 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 21ms, sequenceid=188, compaction requested=true
2024-12-10T15:34:36,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:36,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:36,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:36,954 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:36,955 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 100542 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:36,955 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:36,955 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:36,955 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/132db4ff902a459a948567df93c651dd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=98.2 K
2024-12-10T15:34:36,955 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:36,956 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132db4ff902a459a948567df93c651dd, keycount=54, bloomtype=ROW, size=62.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733844860688
2024-12-10T15:34:36,956 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd6a4719a5946fc861d0e1fed91a97c, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733844874879
2024-12-10T15:34:36,956 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262762f9d0d540fb83224699f485b5d4, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733844876911
2024-12-10T15:34:36,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/bd36401c1943491da721109ce5360acd is 1080, key is row0141/info:/1733844876933/Put/seqid=0
2024-12-10T15:34:36,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741863_1039 (size=17906)
2024-12-10T15:34:36,968 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#79 average throughput is 40.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:36,969 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/817724a52c794abf982d473686a99f66 is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:36,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741863_1039 (size=17906)
2024-12-10T15:34:36,970 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/bd36401c1943491da721109ce5360acd
2024-12-10T15:34:36,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/bd36401c1943491da721109ce5360acd as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd
2024-12-10T15:34:36,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741864_1040 (size=90765)
2024-12-10T15:34:36,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741864_1040 (size=90765)
2024-12-10T15:34:36,980 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd, entries=12, sequenceid=203, filesize=17.5 K
2024-12-10T15:34:36,981 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/817724a52c794abf982d473686a99f66 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/817724a52c794abf982d473686a99f66
2024-12-10T15:34:36,981 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=8.41 KB/8608 for 84446645ea9a814c7eb236a96a7609ff in 26ms, sequenceid=203, compaction requested=false
2024-12-10T15:34:36,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:36,986 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into 817724a52c794abf982d473686a99f66(size=88.6 K), total size for store is 106.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:36,986 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:36,986 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844876954; duration=0sec
2024-12-10T15:34:36,986 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:36,986 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:37,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:37,100 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T15:34:37,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:37,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:38,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:38,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:38,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:38,981 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-10T15:34:38,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ae6bdf82aa3c42b097f2e1f176844fb8 is 1080, key is row0153/info:/1733844876957/Put/seqid=0
2024-12-10T15:34:38,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741865_1041 (size=14672)
2024-12-10T15:34:38,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741865_1041 (size=14672)
2024-12-10T15:34:38,993 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ae6bdf82aa3c42b097f2e1f176844fb8
2024-12-10T15:34:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/ae6bdf82aa3c42b097f2e1f176844fb8 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8
2024-12-10T15:34:39,004 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8, entries=9, sequenceid=216, filesize=14.3 K
2024-12-10T15:34:39,005 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=10.51 KB/10760 for 84446645ea9a814c7eb236a96a7609ff in 25ms, sequenceid=216, compaction requested=true
2024-12-10T15:34:39,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:39,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:39,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:39,005 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:39,007 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-10T15:34:39,007 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:39,007 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:39,007 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:39,007 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/817724a52c794abf982d473686a99f66, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=120.5 K
2024-12-10T15:34:39,007 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 817724a52c794abf982d473686a99f66, keycount=79, bloomtype=ROW, size=88.6 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733844860688
2024-12-10T15:34:39,008 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd36401c1943491da721109ce5360acd, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733844876933
2024-12-10T15:34:39,008 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae6bdf82aa3c42b097f2e1f176844fb8, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733844876957
2024-12-10T15:34:39,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/4c6b183ff64446eebc6f4ac37992729f is 1080, key is row0162/info:/1733844878984/Put/seqid=0
2024-12-10T15:34:39,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741866_1042 (size=16828)
2024-12-10T15:34:39,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741866_1042 (size=16828)
2024-12-10T15:34:39,016 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/4c6b183ff64446eebc6f4ac37992729f
2024-12-10T15:34:39,022 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#82 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:39,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/4c6b183ff64446eebc6f4ac37992729f as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f
2024-12-10T15:34:39,023 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/d82fb8ffc5da42fcba849c6559ebc4bd is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:39,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741867_1043 (size=113509)
2024-12-10T15:34:39,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741867_1043 (size=113509)
2024-12-10T15:34:39,041 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f, entries=11, sequenceid=230, filesize=16.4 K
2024-12-10T15:34:39,042 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 84446645ea9a814c7eb236a96a7609ff in 36ms, sequenceid=230, compaction requested=false
2024-12-10T15:34:39,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:39,044 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-10T15:34:39,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:39,046 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/d82fb8ffc5da42fcba849c6559ebc4bd as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/d82fb8ffc5da42fcba849c6559ebc4bd
2024-12-10T15:34:39,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/cc89cdcb4c3646e1bc1d22228ae6d77d is 1080, key is row0173/info:/1733844879008/Put/seqid=0
2024-12-10T15:34:39,052 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into d82fb8ffc5da42fcba849c6559ebc4bd(size=110.8 K), total size for store is 127.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:39,053 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:39,053 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844879005; duration=0sec
2024-12-10T15:34:39,053 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:39,053 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:39,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741868_1044 (size=14672)
2024-12-10T15:34:39,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741868_1044 (size=14672)
2024-12-10T15:34:39,059 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/cc89cdcb4c3646e1bc1d22228ae6d77d
2024-12-10T15:34:39,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/cc89cdcb4c3646e1bc1d22228ae6d77d as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d
2024-12-10T15:34:39,070 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d, entries=9, sequenceid=242, filesize=14.3 K
2024-12-10T15:34:39,071 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 27ms, sequenceid=242, compaction requested=true
2024-12-10T15:34:39,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:39,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:39,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:39,072 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:39,073 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 145009 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:39,073 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:39,073 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:39,073 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/d82fb8ffc5da42fcba849c6559ebc4bd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=141.6 K
2024-12-10T15:34:39,073 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82fb8ffc5da42fcba849c6559ebc4bd, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733844860688
2024-12-10T15:34:39,074 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6b183ff64446eebc6f4ac37992729f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733844878984
2024-12-10T15:34:39,074 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc89cdcb4c3646e1bc1d22228ae6d77d, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733844879008
2024-12-10T15:34:39,086 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#84 average throughput is 61.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:39,087 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/c38a19a620fa4b2492be5bd2793f4c37 is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:39,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741869_1045 (size=135303)
2024-12-10T15:34:39,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741869_1045 (size=135303)
2024-12-10T15:34:39,095 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/c38a19a620fa4b2492be5bd2793f4c37 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/c38a19a620fa4b2492be5bd2793f4c37
2024-12-10T15:34:39,101 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into c38a19a620fa4b2492be5bd2793f4c37(size=132.1 K), total size for store is 132.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:39,101 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:39,101 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844879071; duration=0sec
2024-12-10T15:34:39,101 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:39,101 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:39,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:39,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:40,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:40,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:40,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:41,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:41,069 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:41,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/5b0e0467b3554088bccac5ca842ddd91 is 1080, key is row0182/info:/1733844879045/Put/seqid=0
2024-12-10T15:34:41,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741870_1046 (size=17907)
2024-12-10T15:34:41,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741870_1046 (size=17907)
2024-12-10T15:34:41,081 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/5b0e0467b3554088bccac5ca842ddd91
2024-12-10T15:34:41,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/5b0e0467b3554088bccac5ca842ddd91 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91
2024-12-10T15:34:41,093 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91, entries=12, sequenceid=259, filesize=17.5 K
2024-12-10T15:34:41,094 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 25ms, sequenceid=259, compaction requested=false
2024-12-10T15:34:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:41,094 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:41,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/91d8dfce8bdc4e6f9a47776e0f83899b is 1080, key is row0194/info:/1733844881070/Put/seqid=0
2024-12-10T15:34:41,103 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741871_1047 (size=17918)
2024-12-10T15:34:41,103 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741871_1047 (size=17918)
2024-12-10T15:34:41,104 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/91d8dfce8bdc4e6f9a47776e0f83899b
2024-12-10T15:34:41,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/91d8dfce8bdc4e6f9a47776e0f83899b as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b
2024-12-10T15:34:41,114 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b, entries=12, sequenceid=274, filesize=17.5 K
2024-12-10T15:34:41,115 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 21ms, sequenceid=274, compaction requested=true
2024-12-10T15:34:41,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:41,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:41,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:41,116 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:41,117 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 171128 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:41,117 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:41,117 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:41,117 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:41,117 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/c38a19a620fa4b2492be5bd2793f4c37, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=167.1 K
2024-12-10T15:34:41,118 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38a19a620fa4b2492be5bd2793f4c37, keycount=120, bloomtype=ROW, size=132.1 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733844860688
2024-12-10T15:34:41,118 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0e0467b3554088bccac5ca842ddd91, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733844879045
2024-12-10T15:34:41,119 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d8dfce8bdc4e6f9a47776e0f83899b, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733844881070
2024-12-10T15:34:41,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83f3a87a60634b30bb565fb1be33223f is 1080, key is row0206/info:/1733844881095/Put/seqid=0
2024-12-10T15:34:41,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741872_1048 (size=17918)
2024-12-10T15:34:41,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741872_1048 (size=17918)
2024-12-10T15:34:41,127 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83f3a87a60634b30bb565fb1be33223f
2024-12-10T15:34:41,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83f3a87a60634b30bb565fb1be33223f as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f
2024-12-10T15:34:41,132 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#88 average throughput is 73.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:41,133 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/80492933dadb4334a43e9f5c33d85c6a is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:41,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741873_1049 (size=161367)
2024-12-10T15:34:41,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741873_1049 (size=161367)
2024-12-10T15:34:41,139 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f, entries=12, sequenceid=289, filesize=17.5 K
2024-12-10T15:34:41,139 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for 84446645ea9a814c7eb236a96a7609ff in 22ms, sequenceid=289, compaction requested=false
2024-12-10T15:34:41,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:41,142 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/80492933dadb4334a43e9f5c33d85c6a as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/80492933dadb4334a43e9f5c33d85c6a
2024-12-10T15:34:41,148 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into 80492933dadb4334a43e9f5c33d85c6a(size=157.6 K), total size for store is 175.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:41,148 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:41,148 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844881115; duration=0sec
2024-12-10T15:34:41,148 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:41,148 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:41,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:41,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:42,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:42,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:42,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:43,049 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:43,142 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-10T15:34:43,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/e8f7b5efac59469f8354694bc6b65c1d is 1080, key is row0218/info:/1733844881118/Put/seqid=0
2024-12-10T15:34:43,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741874_1050 (size=13602)
2024-12-10T15:34:43,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741874_1050 (size=13602)
2024-12-10T15:34:43,156 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/e8f7b5efac59469f8354694bc6b65c1d
2024-12-10T15:34:43,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/e8f7b5efac59469f8354694bc6b65c1d as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d
2024-12-10T15:34:43,166 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d, entries=8, sequenceid=301, filesize=13.3 K
2024-12-10T15:34:43,167 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 26ms, sequenceid=301, compaction requested=true
2024-12-10T15:34:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:43,167 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:43,168 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 192887 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:43,169 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:43,169 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:43,169 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/80492933dadb4334a43e9f5c33d85c6a, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=188.4 K
2024-12-10T15:34:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:43,169 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T15:34:43,169 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80492933dadb4334a43e9f5c33d85c6a, keycount=144, bloomtype=ROW, size=157.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733844860688
2024-12-10T15:34:43,170 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83f3a87a60634b30bb565fb1be33223f, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733844881095
2024-12-10T15:34:43,170 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f7b5efac59469f8354694bc6b65c1d, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733844881118
2024-12-10T15:34:43,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/89c9c711662649ec901083a2d0acb9a7 is 1080, key is row0226/info:/1733844883144/Put/seqid=0
2024-12-10T15:34:43,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741875_1051 (size=17918)
2024-12-10T15:34:43,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741875_1051 (size=17918)
2024-12-10T15:34:43,179 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/89c9c711662649ec901083a2d0acb9a7
2024-12-10T15:34:43,183 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#91 average throughput is 56.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:43,183 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/2b5c53db4691473ab5d4a7341366bd60 is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:43,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/89c9c711662649ec901083a2d0acb9a7 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7
2024-12-10T15:34:43,189 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7, entries=12, sequenceid=316, filesize=17.5 K
2024-12-10T15:34:43,189 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for 84446645ea9a814c7eb236a96a7609ff in 20ms, sequenceid=316, compaction requested=false
2024-12-10T15:34:43,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44815 {}] regionserver.HRegion(8855): Flush requested on 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:43,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-10T15:34:43,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741876_1052 (size=183053)
2024-12-10T15:34:43,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741876_1052 (size=183053)
2024-12-10T15:34:43,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/567cdc138fd5410d9218153bc548dc3a is 1080, key is row0238/info:/1733844883170/Put/seqid=0
2024-12-10T15:34:43,204 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/2b5c53db4691473ab5d4a7341366bd60 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/2b5c53db4691473ab5d4a7341366bd60
2024-12-10T15:34:43,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741877_1053 (size=13602)
2024-12-10T15:34:43,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741877_1053 (size=13602)
2024-12-10T15:34:43,207 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/567cdc138fd5410d9218153bc548dc3a
2024-12-10T15:34:43,211 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into 2b5c53db4691473ab5d4a7341366bd60(size=178.8 K), total size for store is 196.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:43,211 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:43,211 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844883167; duration=0sec
2024-12-10T15:34:43,212 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:43,212 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/567cdc138fd5410d9218153bc548dc3a as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a
2024-12-10T15:34:43,219 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a, entries=8, sequenceid=327, filesize=13.3 K
2024-12-10T15:34:43,220 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for 84446645ea9a814c7eb236a96a7609ff in 29ms, sequenceid=327, compaction requested=true
2024-12-10T15:34:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84446645ea9a814c7eb236a96a7609ff:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T15:34:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:43,220 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T15:34:43,221 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 214573 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T15:34:43,221 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1541): 84446645ea9a814c7eb236a96a7609ff/info is initiating minor compaction (all files)
2024-12-10T15:34:43,221 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84446645ea9a814c7eb236a96a7609ff/info in TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:43,221 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/2b5c53db4691473ab5d4a7341366bd60, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a] into tmpdir=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp, totalSize=209.5 K
2024-12-10T15:34:43,222 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5c53db4691473ab5d4a7341366bd60, keycount=164, bloomtype=ROW, size=178.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733844860688
2024-12-10T15:34:43,222 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c9c711662649ec901083a2d0acb9a7, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733844883144
2024-12-10T15:34:43,223 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567cdc138fd5410d9218153bc548dc3a, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733844883170
2024-12-10T15:34:43,233 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84446645ea9a814c7eb236a96a7609ff#info#compaction#93 average throughput is 62.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T15:34:43,234 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/372820d9f34a4b33a79a2faeb37faa5d is 1080, key is row0062/info:/1733844860688/Put/seqid=0
2024-12-10T15:34:43,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741878_1054 (size=204812)
2024-12-10T15:34:43,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741878_1054 (size=204812)
2024-12-10T15:34:43,246 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/372820d9f34a4b33a79a2faeb37faa5d as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/372820d9f34a4b33a79a2faeb37faa5d
2024-12-10T15:34:43,251 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84446645ea9a814c7eb236a96a7609ff/info of 84446645ea9a814c7eb236a96a7609ff into 372820d9f34a4b33a79a2faeb37faa5d(size=200.0 K), total size for store is 200.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T15:34:43,251 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:43,251 INFO  [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., storeName=84446645ea9a814c7eb236a96a7609ff/info, priority=13, startTime=1733844883220; duration=0sec
2024-12-10T15:34:43,251 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T15:34:43,251 DEBUG [RS:0;d6bcd7308055:44815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84446645ea9a814c7eb236a96a7609ff:info
2024-12-10T15:34:43,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:43,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:44,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:44,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:44,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:45,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:45,214 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-10T15:34:45,215 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44815%2C1733844847291.1733844885214
2024-12-10T15:34:45,223 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,224 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,224 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,224 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,224 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,225 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844847908 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844885214
2024-12-10T15:34:45,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741833_1009 (size=317837)
2024-12-10T15:34:45,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741833_1009 (size=317837)
2024-12-10T15:34:45,233 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39849:39849),(127.0.0.1/127.0.0.1:42873:42873)]
2024-12-10T15:34:45,236 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3c8e97fa70d30e286e3c3552dfd2be3f:

2024-12-10T15:34:45,236 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 84446645ea9a814c7eb236a96a7609ff 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-10T15:34:45,240 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83fbdf168d334a40b79f45d53962032d is 1080, key is row0246/info:/1733844883192/Put/seqid=0
2024-12-10T15:34:45,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741880_1056 (size=16839)
2024-12-10T15:34:45,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741880_1056 (size=16839)
2024-12-10T15:34:45,246 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83fbdf168d334a40b79f45d53962032d
2024-12-10T15:34:45,251 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/.tmp/info/83fbdf168d334a40b79f45d53962032d as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83fbdf168d334a40b79f45d53962032d
2024-12-10T15:34:45,255 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83fbdf168d334a40b79f45d53962032d, entries=11, sequenceid=343, filesize=16.4 K
2024-12-10T15:34:45,256 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 84446645ea9a814c7eb236a96a7609ff in 20ms, sequenceid=343, compaction requested=false
2024-12-10T15:34:45,256 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 84446645ea9a814c7eb236a96a7609ff:

2024-12-10T15:34:45,257 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-12-10T15:34:45,260 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/3a71f936e5694027bd37212563b6c37d is 193, key is TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff./info:regioninfo/1733844863510/Put/seqid=0
2024-12-10T15:34:45,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741881_1057 (size=6223)
2024-12-10T15:34:45,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741881_1057 (size=6223)
2024-12-10T15:34:45,264 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/3a71f936e5694027bd37212563b6c37d
2024-12-10T15:34:45,268 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/.tmp/info/3a71f936e5694027bd37212563b6c37d as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/info/3a71f936e5694027bd37212563b6c37d
2024-12-10T15:34:45,273 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/info/3a71f936e5694027bd37212563b6c37d, entries=5, sequenceid=21, filesize=6.1 K
2024-12-10T15:34:45,273 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 17ms, sequenceid=21, compaction requested=false
2024-12-10T15:34:45,274 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T15:34:45,274 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44815%2C1733844847291.1733844885274
2024-12-10T15:34:45,280 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,280 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,280 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,280 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,280 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,280 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844885214 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844885274
2024-12-10T15:34:45,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741879_1055 (size=731)
2024-12-10T15:34:45,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741879_1055 (size=731)
2024-12-10T15:34:45,285 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844847908 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs/d6bcd7308055%2C44815%2C1733844847291.1733844847908
2024-12-10T15:34:45,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39849:39849),(127.0.0.1/127.0.0.1:42873:42873)]
2024-12-10T15:34:45,285 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-10T15:34:45,286 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:34:45,286 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/WALs/d6bcd7308055,44815,1733844847291/d6bcd7308055%2C44815%2C1733844847291.1733844885214 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs/d6bcd7308055%2C44815%2C1733844847291.1733844885214
2024-12-10T15:34:45,286 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:45,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:45,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:45,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:45,286 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:34:45,286 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:34:45,286 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143909579, stopped=false
2024-12-10T15:34:45,286 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,35351,1733844847117
2024-12-10T15:34:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:45,341 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:45,342 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:45,342 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:45,342 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:45,342 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:45,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:45,343 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,44815,1733844847291' *****
2024-12-10T15:34:45,343 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:34:45,343 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c8e97fa70d30e286e3c3552dfd2be3f
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(3091): Received CLOSE for 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,44815,1733844847291
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:44815.
2024-12-10T15:34:45,343 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c8e97fa70d30e286e3c3552dfd2be3f, disabling compactions & flushes
2024-12-10T15:34:45,343 DEBUG [RS:0;d6bcd7308055:44815 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:45,343 DEBUG [RS:0;d6bcd7308055:44815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:45,343 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:45,343 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:45,343 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f. after waiting 0 ms
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:34:45,343 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:34:45,343 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:34:45,344 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:34:45,344 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-10T15:34:45,344 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1325): Online Regions={3c8e97fa70d30e286e3c3552dfd2be3f=TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f., 84446645ea9a814c7eb236a96a7609ff=TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff., 1588230740=hbase:meta,,1.1588230740}
2024-12-10T15:34:45,344 DEBUG [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3c8e97fa70d30e286e3c3552dfd2be3f, 84446645ea9a814c7eb236a96a7609ff
2024-12-10T15:34:45,344 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:34:45,344 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:34:45,344 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:34:45,344 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:34:45,344 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:34:45,344 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-bottom] to archive
2024-12-10T15:34:45,345 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:34:45,346 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:45,346 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d6bcd7308055:35351 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T15:34:45,347 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-10T15:34:45,348 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-10T15:34:45,349 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:34:45,349 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:45,349 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844885344Running coprocessor pre-close hooks at 1733844885344Disabling compacts and flushes for region at 1733844885344Disabling writes for close at 1733844885344Writing region close event to WAL at 1733844885345 (+1 ms)Running coprocessor post-close hooks at 1733844885349 (+4 ms)Closed at 1733844885349
2024-12-10T15:34:45,349 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/3c8e97fa70d30e286e3c3552dfd2be3f/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-12-10T15:34:45,350 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c8e97fa70d30e286e3c3552dfd2be3f:
Waiting for close lock at 1733844885343Running coprocessor pre-close hooks at 1733844885343Disabling compacts and flushes for region at 1733844885343Disabling writes for close at 1733844885343Writing region close event to WAL at 1733844885347 (+4 ms)Running coprocessor post-close hooks at 1733844885350 (+3 ms)Closed at 1733844885350
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733844862808.3c8e97fa70d30e286e3c3552dfd2be3f.
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 84446645ea9a814c7eb236a96a7609ff, disabling compactions & flushes
2024-12-10T15:34:45,350 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:45,350 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff. after waiting 0 ms
2024-12-10T15:34:45,351 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:45,351 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0->hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/fb2b710463a06c67c23fb6d8011bdfd0/info/cfbc9fd651f34d62a0e9c8398284f285-top, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ecf9714275bd4046acd5f7fa7819337b, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/132db4ff902a459a948567df93c651dd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/817724a52c794abf982d473686a99f66, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/d82fb8ffc5da42fcba849c6559ebc4bd, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/c38a19a620fa4b2492be5bd2793f4c37, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/80492933dadb4334a43e9f5c33d85c6a, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/2b5c53db4691473ab5d4a7341366bd60, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a] to archive
2024-12-10T15:34:45,352 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1ac96179c7fc4cefb5fc5e8e660d4ed1
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cfbc9fd651f34d62a0e9c8398284f285.fb2b710463a06c67c23fb6d8011bdfd0
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ecf9714275bd4046acd5f7fa7819337b to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ecf9714275bd4046acd5f7fa7819337b
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/132db4ff902a459a948567df93c651dd to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/132db4ff902a459a948567df93c651dd
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/1cd6a4719a5946fc861d0e1fed91a97c
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ac132cbd1f9b4d13a9ff247582e5c7cc
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-96b428454a1c42c8b1857d084bd395ea
2024-12-10T15:34:45,354 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/TestLogRolling-testLogRolling=fb2b710463a06c67c23fb6d8011bdfd0-f65bdd695fd5452cbf5d2522cdc4dab7
2024-12-10T15:34:45,355 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/262762f9d0d540fb83224699f485b5d4
2024-12-10T15:34:45,356 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/ae6bdf82aa3c42b097f2e1f176844fb8
2024-12-10T15:34:45,356 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/d82fb8ffc5da42fcba849c6559ebc4bd to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/d82fb8ffc5da42fcba849c6559ebc4bd
2024-12-10T15:34:45,356 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/bd36401c1943491da721109ce5360acd
2024-12-10T15:34:45,356 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/4c6b183ff64446eebc6f4ac37992729f
2024-12-10T15:34:45,357 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/c38a19a620fa4b2492be5bd2793f4c37 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/c38a19a620fa4b2492be5bd2793f4c37
2024-12-10T15:34:45,357 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/cc89cdcb4c3646e1bc1d22228ae6d77d
2024-12-10T15:34:45,357 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/817724a52c794abf982d473686a99f66 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/817724a52c794abf982d473686a99f66
2024-12-10T15:34:45,357 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/80492933dadb4334a43e9f5c33d85c6a to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/80492933dadb4334a43e9f5c33d85c6a
2024-12-10T15:34:45,358 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/5b0e0467b3554088bccac5ca842ddd91
2024-12-10T15:34:45,358 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/83f3a87a60634b30bb565fb1be33223f
2024-12-10T15:34:45,358 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/91d8dfce8bdc4e6f9a47776e0f83899b
2024-12-10T15:34:45,358 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/2b5c53db4691473ab5d4a7341366bd60 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/2b5c53db4691473ab5d4a7341366bd60
2024-12-10T15:34:45,359 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/567cdc138fd5410d9218153bc548dc3a
2024-12-10T15:34:45,359 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/e8f7b5efac59469f8354694bc6b65c1d
2024-12-10T15:34:45,359 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7 to hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/archive/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/info/89c9c711662649ec901083a2d0acb9a7
2024-12-10T15:34:45,359 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ecf9714275bd4046acd5f7fa7819337b=42984, 1ac96179c7fc4cefb5fc5e8e660d4ed1=12516, 132db4ff902a459a948567df93c651dd=63636, ac132cbd1f9b4d13a9ff247582e5c7cc=17906, 1cd6a4719a5946fc861d0e1fed91a97c=20078, 817724a52c794abf982d473686a99f66=90765, 262762f9d0d540fb83224699f485b5d4=16828, bd36401c1943491da721109ce5360acd=17906, d82fb8ffc5da42fcba849c6559ebc4bd=113509, ae6bdf82aa3c42b097f2e1f176844fb8=14672, 4c6b183ff64446eebc6f4ac37992729f=16828, c38a19a620fa4b2492be5bd2793f4c37=135303, cc89cdcb4c3646e1bc1d22228ae6d77d=14672, 5b0e0467b3554088bccac5ca842ddd91=17907, 80492933dadb4334a43e9f5c33d85c6a=161367, 91d8dfce8bdc4e6f9a47776e0f83899b=17918, 83f3a87a60634b30bb565fb1be33223f=17918, 2b5c53db4691473ab5d4a7341366bd60=183053, e8f7b5efac59469f8354694bc6b65c1d=13602, 89c9c711662649ec901083a2d0acb9a7=17918, 567cdc138fd5410d9218153bc548dc3a=13602]
2024-12-10T15:34:45,362 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/data/default/TestLogRolling-testLogRolling/84446645ea9a814c7eb236a96a7609ff/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130
2024-12-10T15:34:45,363 INFO  [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:45,363 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 84446645ea9a814c7eb236a96a7609ff:
Waiting for close lock at 1733844885350Running coprocessor pre-close hooks at 1733844885350Disabling compacts and flushes for region at 1733844885350Disabling writes for close at 1733844885351 (+1 ms)Writing region close event to WAL at 1733844885359 (+8 ms)Running coprocessor post-close hooks at 1733844885363 (+4 ms)Closed at 1733844885363
2024-12-10T15:34:45,363 DEBUG [RS_CLOSE_REGION-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733844862808.84446645ea9a814c7eb236a96a7609ff.
2024-12-10T15:34:45,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:45,544 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,44815,1733844847291; all regions closed.
2024-12-10T15:34:45,544 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,545 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,545 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,545 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,545 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741834_1010 (size=8107)
2024-12-10T15:34:45,548 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741834_1010 (size=8107)
2024-12-10T15:34:45,550 DEBUG [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs
2024-12-10T15:34:45,550 INFO  [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44815%2C1733844847291.meta:.meta(num 1733844848343)
2024-12-10T15:34:45,551 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,551 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,551 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,551 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,551 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741882_1058 (size=780)
2024-12-10T15:34:45,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741882_1058 (size=780)
2024-12-10T15:34:45,555 DEBUG [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/oldWALs
2024-12-10T15:34:45,555 INFO  [RS:0;d6bcd7308055:44815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44815%2C1733844847291:(num 1733844885274)
2024-12-10T15:34:45,555 DEBUG [RS:0;d6bcd7308055:44815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:45,555 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:34:45,556 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:45,556 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:45,556 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:45,556 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:45,556 INFO  [RS:0;d6bcd7308055:44815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44815
2024-12-10T15:34:45,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,44815,1733844847291
2024-12-10T15:34:45,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:34:45,562 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:45,573 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,44815,1733844847291]
2024-12-10T15:34:45,584 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,44815,1733844847291 already deleted, retry=false
2024-12-10T15:34:45,584 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,44815,1733844847291 expired; onlineServers=0
2024-12-10T15:34:45,584 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,35351,1733844847117' *****
2024-12-10T15:34:45,584 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:34:45,584 INFO  [M:0;d6bcd7308055:35351 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:45,584 INFO  [M:0;d6bcd7308055:35351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:45,584 DEBUG [M:0;d6bcd7308055:35351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:34:45,584 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:34:45,585 DEBUG [M:0;d6bcd7308055:35351 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:34:45,585 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844847637 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844847637,5,FailOnTimeoutGroup]
2024-12-10T15:34:45,585 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844847640 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844847640,5,FailOnTimeoutGroup]
2024-12-10T15:34:45,585 INFO  [M:0;d6bcd7308055:35351 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:45,585 INFO  [M:0;d6bcd7308055:35351 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:45,585 DEBUG [M:0;d6bcd7308055:35351 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:34:45,586 INFO  [M:0;d6bcd7308055:35351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:34:45,586 INFO  [M:0;d6bcd7308055:35351 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:45,586 INFO  [M:0;d6bcd7308055:35351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:34:45,586 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:34:45,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:34:45,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:45,594 DEBUG [M:0;d6bcd7308055:35351 {}] zookeeper.ZKUtil(347): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T15:34:45,595 WARN  [M:0;d6bcd7308055:35351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T15:34:45,596 INFO  [M:0;d6bcd7308055:35351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/.lastflushedseqids
2024-12-10T15:34:45,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741883_1059 (size=228)
2024-12-10T15:34:45,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741883_1059 (size=228)
2024-12-10T15:34:45,606 INFO  [M:0;d6bcd7308055:35351 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:34:45,607 INFO  [M:0;d6bcd7308055:35351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:34:45,607 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:34:45,607 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:45,607 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:45,607 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:34:45,607 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:45,608 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=53.70 KB heapSize=65.93 KB
2024-12-10T15:34:45,627 DEBUG [M:0;d6bcd7308055:35351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2be76c0fb0ae4ee9819de70bf6cc5a34 is 82, key is hbase:meta,,1/info:regioninfo/1733844848369/Put/seqid=0
2024-12-10T15:34:45,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741884_1060 (size=5672)
2024-12-10T15:34:45,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741884_1060 (size=5672)
2024-12-10T15:34:45,632 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2be76c0fb0ae4ee9819de70bf6cc5a34
2024-12-10T15:34:45,650 DEBUG [M:0;d6bcd7308055:35351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3026c8131a094dad99473119c2763f28 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733844848912/Put/seqid=0
2024-12-10T15:34:45,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741885_1061 (size=7679)
2024-12-10T15:34:45,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741885_1061 (size=7679)
2024-12-10T15:34:45,655 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.09 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3026c8131a094dad99473119c2763f28
2024-12-10T15:34:45,658 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3026c8131a094dad99473119c2763f28
2024-12-10T15:34:45,671 DEBUG [M:0;d6bcd7308055:35351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f04e9b36ac034e8582d0af213ef8fcc8 is 69, key is d6bcd7308055,44815,1733844847291/rs:state/1733844847755/Put/seqid=0
2024-12-10T15:34:45,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:45,673 INFO  [RS:0;d6bcd7308055:44815 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:45,673 INFO  [RS:0;d6bcd7308055:44815 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,44815,1733844847291; zookeeper connection closed.
2024-12-10T15:34:45,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44815-0x100105cd0c40001, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:45,674 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b264d7d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b264d7d
2024-12-10T15:34:45,674 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:34:45,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741886_1062 (size=5156)
2024-12-10T15:34:45,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741886_1062 (size=5156)
2024-12-10T15:34:45,675 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f04e9b36ac034e8582d0af213ef8fcc8
2024-12-10T15:34:45,690 DEBUG [M:0;d6bcd7308055:35351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fc08c868bb24d4bbac2b29e3c681373 is 52, key is load_balancer_on/state:d/1733844848521/Put/seqid=0
2024-12-10T15:34:45,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741887_1063 (size=5056)
2024-12-10T15:34:45,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741887_1063 (size=5056)
2024-12-10T15:34:45,694 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fc08c868bb24d4bbac2b29e3c681373
2024-12-10T15:34:45,698 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2be76c0fb0ae4ee9819de70bf6cc5a34 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2be76c0fb0ae4ee9819de70bf6cc5a34
2024-12-10T15:34:45,704 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2be76c0fb0ae4ee9819de70bf6cc5a34, entries=8, sequenceid=129, filesize=5.5 K
2024-12-10T15:34:45,705 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3026c8131a094dad99473119c2763f28 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3026c8131a094dad99473119c2763f28
2024-12-10T15:34:45,708 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3026c8131a094dad99473119c2763f28
2024-12-10T15:34:45,708 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3026c8131a094dad99473119c2763f28, entries=14, sequenceid=129, filesize=7.5 K
2024-12-10T15:34:45,709 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f04e9b36ac034e8582d0af213ef8fcc8 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f04e9b36ac034e8582d0af213ef8fcc8
2024-12-10T15:34:45,713 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f04e9b36ac034e8582d0af213ef8fcc8, entries=1, sequenceid=129, filesize=5.0 K
2024-12-10T15:34:45,713 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fc08c868bb24d4bbac2b29e3c681373 as hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fc08c868bb24d4bbac2b29e3c681373
2024-12-10T15:34:45,717 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40063/user/jenkins/test-data/471264cd-13c5-3432-9bbc-ee7b8833e9b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fc08c868bb24d4bbac2b29e3c681373, entries=1, sequenceid=129, filesize=4.9 K
2024-12-10T15:34:45,718 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.70 KB/54985, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=129, compaction requested=false
2024-12-10T15:34:45,719 INFO  [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:45,719 DEBUG [M:0;d6bcd7308055:35351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844885607Disabling compacts and flushes for region at 1733844885607Disabling writes for close at 1733844885607Obtaining lock to block concurrent updates at 1733844885608 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844885608Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=54985, getHeapSize=67448, getOffHeapSize=0, getCellsCount=152 at 1733844885608Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844885610 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844885610Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844885627 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844885627Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844885636 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844885649 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844885649Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844885659 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844885670 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844885670Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844885679 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844885690 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844885690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c55208b: reopening flushed file at 1733844885698 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58e7e420: reopening flushed file at 1733844885704 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f511562: reopening flushed file at 1733844885709 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60226490: reopening flushed file at 1733844885713 (+4 ms)Finished flush of dataSize ~53.70 KB/54985, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=129, compaction requested=false at 1733844885718 (+5 ms)Writing region close event to WAL at 1733844885719 (+1 ms)Closed at 1733844885719
2024-12-10T15:34:45,719 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,720 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,720 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,720 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,720 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:45,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:45,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45095 is added to blk_1073741830_1006 (size=63915)
2024-12-10T15:34:45,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46355 is added to blk_1073741830_1006 (size=63915)
2024-12-10T15:34:45,722 INFO  [M:0;d6bcd7308055:35351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:34:45,722 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:45,722 INFO  [M:0;d6bcd7308055:35351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35351
2024-12-10T15:34:45,722 INFO  [M:0;d6bcd7308055:35351 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:45,778 INFO  [regionserver/d6bcd7308055:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:34:45,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:45,831 INFO  [M:0;d6bcd7308055:35351 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:45,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35351-0x100105cd0c40000, quorum=127.0.0.1:64316, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:45,834 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25d8f267{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:45,835 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7aa18c0f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:45,835 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:45,835 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@206e2def{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:45,835 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@426b898{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:45,838 WARN  [BP-1146057193-172.17.0.2-1733844844878 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:45,838 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:45,838 WARN  [BP-1146057193-172.17.0.2-1733844844878 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1146057193-172.17.0.2-1733844844878 (Datanode Uuid fbdd1eb9-c22e-4500-9333-5668f2fc13e8) service to localhost/127.0.0.1:40063
2024-12-10T15:34:45,838 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:45,839 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data3/current/BP-1146057193-172.17.0.2-1733844844878 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:45,839 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data4/current/BP-1146057193-172.17.0.2-1733844844878 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:45,839 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:45,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58368375{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:45,843 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@338baa7b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:45,843 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:45,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42a7c471{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:45,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fdb83bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:45,845 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:45,845 WARN  [BP-1146057193-172.17.0.2-1733844844878 heartbeating to localhost/127.0.0.1:40063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:45,845 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:45,845 WARN  [BP-1146057193-172.17.0.2-1733844844878 heartbeating to localhost/127.0.0.1:40063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1146057193-172.17.0.2-1733844844878 (Datanode Uuid cd57e13b-228c-44a3-a7d1-01424a7febbb) service to localhost/127.0.0.1:40063
2024-12-10T15:34:45,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data1/current/BP-1146057193-172.17.0.2-1733844844878 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:45,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/cluster_8fc91cf5-e471-f620-0cd7-03e47cb0af8e/data/data2/current/BP-1146057193-172.17.0.2-1733844844878 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:45,845 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:45,850 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4b7ab4bd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:34:45,851 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2fcc97ec{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:45,851 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:45,851 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32e62001{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:45,851 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e2d8e1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:45,858 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:34:45,887 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:34:45,897 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=240 (was 212)
Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40063 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:40063
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40063
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-16
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40063 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:40063 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HFileArchiver-15
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-18
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40063
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-17
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=512 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=7212 (was 7265)
2024-12-10T15:34:45,904 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=240, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=7213
2024-12-10T15:34:45,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T15:34:45,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.log.dir so I do NOT create it in target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df
2024-12-10T15:34:45,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0e1728f9-83cd-6338-b15d-9f7ada4a4a5b/hadoop.tmp.dir so I do NOT create it in target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c, deleteOnExit=true
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/test.cache.data in system properties and HBase conf
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir in system properties and HBase conf
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T15:34:45,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T15:34:45,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/nfs.dump.dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/java.io.tmpdir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T15:34:45,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T15:34:45,919 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:34:46,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:46,198 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:46,201 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:46,202 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:46,202 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:46,202 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T15:34:46,202 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:46,203 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48174132{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:46,203 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@117d3f54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:46,289 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@69ffafd5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/java.io.tmpdir/jetty-localhost-46635-hadoop-hdfs-3_4_1-tests_jar-_-any-16218334444087447010/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:34:46,289 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29afca44{HTTP/1.1, (http/1.1)}{localhost:46635}
2024-12-10T15:34:46,289 INFO  [Time-limited test {}] server.Server(415): Started @300880ms
2024-12-10T15:34:46,299 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T15:34:46,540 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:46,542 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:46,543 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:46,543 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:46,543 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:34:46,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:46,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1194c52d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:46,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2685eb00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:46,633 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43d2c6e8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/java.io.tmpdir/jetty-localhost-41823-hadoop-hdfs-3_4_1-tests_jar-_-any-6970983333453020281/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:46,633 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@55c02843{HTTP/1.1, (http/1.1)}{localhost:41823}
2024-12-10T15:34:46,633 INFO  [Time-limited test {}] server.Server(415): Started @301224ms
2024-12-10T15:34:46,634 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:34:46,660 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T15:34:46,662 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T15:34:46,662 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T15:34:46,663 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T15:34:46,663 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T15:34:46,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f89e80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,AVAILABLE}
2024-12-10T15:34:46,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cde2eb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T15:34:46,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:46,751 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ed6212e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/java.io.tmpdir/jetty-localhost-42309-hadoop-hdfs-3_4_1-tests_jar-_-any-11639622380848302914/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:46,751 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3593a6d0{HTTP/1.1, (http/1.1)}{localhost:42309}
2024-12-10T15:34:46,751 INFO  [Time-limited test {}] server.Server(415): Started @301342ms
2024-12-10T15:34:46,752 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T15:34:47,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:47,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:47,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:47,809 WARN  [Thread-2514 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data1/current/BP-2090031351-172.17.0.2-1733844885923/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:47,809 WARN  [Thread-2515 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data2/current/BP-2090031351-172.17.0.2-1733844885923/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:47,827 WARN  [Thread-2478 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:34:47,829 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x933f19503a729eb8 with lease ID 0xea0f7b1da33903f8: Processing first storage report for DS-8eaf6965-50fd-4896-8529-a5a17a0e9ea1 from datanode DatanodeRegistration(127.0.0.1:42797, datanodeUuid=661dabbf-646d-4d4e-913d-150c73bc172e, infoPort=45913, infoSecurePort=0, ipcPort=41329, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923)
2024-12-10T15:34:47,829 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x933f19503a729eb8 with lease ID 0xea0f7b1da33903f8: from storage DS-8eaf6965-50fd-4896-8529-a5a17a0e9ea1 node DatanodeRegistration(127.0.0.1:42797, datanodeUuid=661dabbf-646d-4d4e-913d-150c73bc172e, infoPort=45913, infoSecurePort=0, ipcPort=41329, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:47,829 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x933f19503a729eb8 with lease ID 0xea0f7b1da33903f8: Processing first storage report for DS-636c1d4f-1041-42da-869a-8f7af600a228 from datanode DatanodeRegistration(127.0.0.1:42797, datanodeUuid=661dabbf-646d-4d4e-913d-150c73bc172e, infoPort=45913, infoSecurePort=0, ipcPort=41329, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923)
2024-12-10T15:34:47,829 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x933f19503a729eb8 with lease ID 0xea0f7b1da33903f8: from storage DS-636c1d4f-1041-42da-869a-8f7af600a228 node DatanodeRegistration(127.0.0.1:42797, datanodeUuid=661dabbf-646d-4d4e-913d-150c73bc172e, infoPort=45913, infoSecurePort=0, ipcPort=41329, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T15:34:47,952 WARN  [Thread-2525 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data3/current/BP-2090031351-172.17.0.2-1733844885923/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:47,952 WARN  [Thread-2526 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data4/current/BP-2090031351-172.17.0.2-1733844885923/current, will proceed with Du for space computation calculation, 
2024-12-10T15:34:47,975 WARN  [Thread-2501 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T15:34:47,977 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbfb880eaa6fad259 with lease ID 0xea0f7b1da33903f9: Processing first storage report for DS-c4fa4398-8077-463c-b67c-7028681ee230 from datanode DatanodeRegistration(127.0.0.1:44905, datanodeUuid=32584107-aadf-4dde-a866-8777bc8e5832, infoPort=45969, infoSecurePort=0, ipcPort=44505, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923)
2024-12-10T15:34:47,977 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbfb880eaa6fad259 with lease ID 0xea0f7b1da33903f9: from storage DS-c4fa4398-8077-463c-b67c-7028681ee230 node DatanodeRegistration(127.0.0.1:44905, datanodeUuid=32584107-aadf-4dde-a866-8777bc8e5832, infoPort=45969, infoSecurePort=0, ipcPort=44505, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:47,977 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbfb880eaa6fad259 with lease ID 0xea0f7b1da33903f9: Processing first storage report for DS-63c9ded5-c454-4d38-a758-103795b1d81f from datanode DatanodeRegistration(127.0.0.1:44905, datanodeUuid=32584107-aadf-4dde-a866-8777bc8e5832, infoPort=45969, infoSecurePort=0, ipcPort=44505, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923)
2024-12-10T15:34:47,977 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbfb880eaa6fad259 with lease ID 0xea0f7b1da33903f9: from storage DS-63c9ded5-c454-4d38-a758-103795b1d81f node DatanodeRegistration(127.0.0.1:44905, datanodeUuid=32584107-aadf-4dde-a866-8777bc8e5832, infoPort=45969, infoSecurePort=0, ipcPort=44505, storageInfo=lv=-57;cid=testClusterID;nsid=274623555;c=1733844885923), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T15:34:47,981 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df
2024-12-10T15:34:48,005 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/zookeeper_0, clientPort=60588, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T15:34:48,006 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60588
2024-12-10T15:34:48,006 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,007 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:34:48,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741825_1001 (size=7)
2024-12-10T15:34:48,016 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad with version=8
2024-12-10T15:34:48,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:44521/user/jenkins/test-data/71618d57-e534-45f7-caf5-288f1691758a/hbase-staging
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T15:34:48,018 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:34:48,019 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34879
2024-12-10T15:34:48,019 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34879 connecting to ZooKeeper ensemble=127.0.0.1:60588
2024-12-10T15:34:48,053 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:48,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:348790x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:34:48,171 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34879-0x100105d708b0000 connected
2024-12-10T15:34:48,331 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,333 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:48,335 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad, hbase.cluster.distributed=false
2024-12-10T15:34:48,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:34:48,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34879
2024-12-10T15:34:48,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34879
2024-12-10T15:34:48,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34879
2024-12-10T15:34:48,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34879
2024-12-10T15:34:48,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34879
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6bcd7308055:0 server-side Connection retries=45
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T15:34:48,358 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T15:34:48,359 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44261
2024-12-10T15:34:48,359 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44261 connecting to ZooKeeper ensemble=127.0.0.1:60588
2024-12-10T15:34:48,360 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,361 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442610x0, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T15:34:48,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44261-0x100105d708b0001 connected
2024-12-10T15:34:48,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:48,374 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T15:34:48,374 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T15:34:48,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T15:34:48,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T15:34:48,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44261
2024-12-10T15:34:48,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44261
2024-12-10T15:34:48,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44261
2024-12-10T15:34:48,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44261
2024-12-10T15:34:48,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44261
2024-12-10T15:34:48,390 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6bcd7308055:34879
2024-12-10T15:34:48,391 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:48,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:48,394 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T15:34:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,405 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T15:34:48,406 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6bcd7308055,34879,1733844888017 from backup master directory
2024-12-10T15:34:48,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:48,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T15:34:48,415 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:34:48,415 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/hbase.id] with ID: d7de8344-98fc-4665-8a72-ff1f87faa6cd
2024-12-10T15:34:48,418 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/.tmp/hbase.id
2024-12-10T15:34:48,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:34:48,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741826_1002 (size=42)
2024-12-10T15:34:48,423 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/.tmp/hbase.id]:[hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/hbase.id]
2024-12-10T15:34:48,437 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:48,437 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T15:34:48,438 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T15:34:48,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:34:48,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741827_1003 (size=196)
2024-12-10T15:34:48,452 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T15:34:48,453 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T15:34:48,453 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:48,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:34:48,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741828_1004 (size=1189)
2024-12-10T15:34:48,459 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store
2024-12-10T15:34:48,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:34:48,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741829_1005 (size=34)
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:34:48,465 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:48,465 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:48,465 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844888465Disabling compacts and flushes for region at 1733844888465Disabling writes for close at 1733844888465Writing region close event to WAL at 1733844888465Closed at 1733844888465
2024-12-10T15:34:48,465 WARN  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/.initializing
2024-12-10T15:34:48,466 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/WALs/d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,467 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C34879%2C1733844888017, suffix=, logDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/WALs/d6bcd7308055,34879,1733844888017, archiveDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/oldWALs, maxLogs=10
2024-12-10T15:34:48,468 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C34879%2C1733844888017.1733844888468
2024-12-10T15:34:48,471 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/WALs/d6bcd7308055,34879,1733844888017/d6bcd7308055%2C34879%2C1733844888017.1733844888468
2024-12-10T15:34:48,472 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45969:45969),(127.0.0.1/127.0.0.1:45913:45913)]
2024-12-10T15:34:48,477 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:34:48,477 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:48,477 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,477 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,478 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:48,479 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T15:34:48,479 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,480 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:48,480 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:48,481 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T15:34:48,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,481 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:48,482 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:48,483 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T15:34:48,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,483 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:48,483 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T15:34:48,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T15:34:48,484 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T15:34:48,484 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,485 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,485 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,486 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,486 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,487 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T15:34:48,487 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T15:34:48,489 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:34:48,489 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=721750, jitterRate=-0.08224846422672272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T15:34:48,490 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733844888477Initializing all the Stores at 1733844888478 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844888478Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844888478Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844888478Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844888478Cleaning up temporary data from old regions at 1733844888486 (+8 ms)Region opened successfully at 1733844888490 (+4 ms)
2024-12-10T15:34:48,490 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T15:34:48,492 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f560eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T15:34:48,493 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T15:34:48,495 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T15:34:48,496 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T15:34:48,499 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T15:34:48,499 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T15:34:48,500 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T15:34:48,509 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T15:34:48,510 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T15:34:48,511 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T15:34:48,520 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T15:34:48,521 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T15:34:48,530 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T15:34:48,533 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T15:34:48,541 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T15:34:48,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,634 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6bcd7308055,34879,1733844888017, sessionid=0x100105d708b0000, setting cluster-up flag (Was=false)
2024-12-10T15:34:48,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,699 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T15:34:48,703 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:48,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:48,752 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T15:34:48,756 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6bcd7308055,34879,1733844888017
2024-12-10T15:34:48,759 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T15:34:48,762 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:48,762 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T15:34:48,762 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T15:34:48,763 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6bcd7308055,34879,1733844888017
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6bcd7308055:0, corePoolSize=5, maxPoolSize=5
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6bcd7308055:0, corePoolSize=10, maxPoolSize=10
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:34:48,766 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,767 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733844918767
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:48,768 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T15:34:48,768 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T15:34:48,769 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844888769,5,FailOnTimeoutGroup]
2024-12-10T15:34:48,769 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844888769,5,FailOnTimeoutGroup]
2024-12-10T15:34:48,769 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,769 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,769 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T15:34:48,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:34:48,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741831_1007 (size=1321)
2024-12-10T15:34:48,775 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T15:34:48,775 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad
2024-12-10T15:34:48,779 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(746): ClusterId : d7de8344-98fc-4665-8a72-ff1f87faa6cd
2024-12-10T15:34:48,780 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T15:34:48,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:34:48,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741832_1008 (size=32)
2024-12-10T15:34:48,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:48,783 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:34:48,784 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T15:34:48,784 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T15:34:48,785 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:34:48,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:48,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:34:48,787 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:34:48,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,787 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:48,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:34:48,789 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:34:48,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,789 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:48,789 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:34:48,790 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:34:48,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:48,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:48,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:34:48,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740
2024-12-10T15:34:48,792 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740
2024-12-10T15:34:48,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:34:48,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:34:48,794 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:34:48,794 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T15:34:48,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:34:48,795 DEBUG [RS:0;d6bcd7308055:44261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ae3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6bcd7308055/172.17.0.2:0
2024-12-10T15:34:48,796 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T15:34:48,797 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754494, jitterRate=-0.04061137139797211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:34:48,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733844888782Initializing all the Stores at 1733844888783 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844888783Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844888783Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844888783Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844888783Cleaning up temporary data from old regions at 1733844888793 (+10 ms)Region opened successfully at 1733844888797 (+4 ms)
2024-12-10T15:34:48,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:34:48,797 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:34:48,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:34:48,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:34:48,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:34:48,798 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:48,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844888797Disabling compacts and flushes for region at 1733844888797Disabling writes for close at 1733844888797Writing region close event to WAL at 1733844888798 (+1 ms)Closed at 1733844888798
2024-12-10T15:34:48,799 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:48,799 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T15:34:48,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T15:34:48,800 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:34:48,800 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T15:34:48,805 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6bcd7308055:44261
2024-12-10T15:34:48,805 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T15:34:48,805 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T15:34:48,805 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T15:34:48,806 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6bcd7308055,34879,1733844888017 with port=44261, startcode=1733844888357
2024-12-10T15:34:48,806 DEBUG [RS:0;d6bcd7308055:44261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T15:34:48,807 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59435, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T15:34:48,808 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34879 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,808 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34879 {}] master.ServerManager(517): Registering regionserver=d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,809 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad
2024-12-10T15:34:48,809 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46409
2024-12-10T15:34:48,809 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T15:34:48,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:34:48,815 DEBUG [RS:0;d6bcd7308055:44261 {}] zookeeper.ZKUtil(111): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,815 WARN  [RS:0;d6bcd7308055:44261 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T15:34:48,815 INFO  [RS:0;d6bcd7308055:44261 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:48,815 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,815 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6bcd7308055,44261,1733844888357]
2024-12-10T15:34:48,818 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T15:34:48,820 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T15:34:48,820 INFO  [RS:0;d6bcd7308055:44261 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T15:34:48,820 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,820 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T15:34:48,821 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T15:34:48,821 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6bcd7308055:0, corePoolSize=2, maxPoolSize=2
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6bcd7308055:0, corePoolSize=1, maxPoolSize=1
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:34:48,821 DEBUG [RS:0;d6bcd7308055:44261 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6bcd7308055:0, corePoolSize=3, maxPoolSize=3
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,825 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44261,1733844888357-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:34:48,837 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T15:34:48,837 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,44261,1733844888357-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,837 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,837 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.Replication(171): d6bcd7308055,44261,1733844888357 started
2024-12-10T15:34:48,848 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:48,848 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1482): Serving as d6bcd7308055,44261,1733844888357, RpcServer on d6bcd7308055/172.17.0.2:44261, sessionid=0x100105d708b0001
2024-12-10T15:34:48,848 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T15:34:48,848 DEBUG [RS:0;d6bcd7308055:44261 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,848 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44261,1733844888357'
2024-12-10T15:34:48,848 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6bcd7308055,44261,1733844888357
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6bcd7308055,44261,1733844888357'
2024-12-10T15:34:48,849 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T15:34:48,850 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T15:34:48,850 DEBUG [RS:0;d6bcd7308055:44261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T15:34:48,850 INFO  [RS:0;d6bcd7308055:44261 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T15:34:48,850 INFO  [RS:0;d6bcd7308055:44261 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T15:34:48,951 WARN  [d6bcd7308055:34879 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T15:34:48,953 INFO  [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44261%2C1733844888357, suffix=, logDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/d6bcd7308055,44261,1733844888357, archiveDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs, maxLogs=32
2024-12-10T15:34:48,954 INFO  [RS:0;d6bcd7308055:44261 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44261%2C1733844888357.1733844888954
2024-12-10T15:34:48,963 INFO  [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/d6bcd7308055,44261,1733844888357/d6bcd7308055%2C44261%2C1733844888357.1733844888954
2024-12-10T15:34:48,969 DEBUG [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45913:45913),(127.0.0.1/127.0.0.1:45969:45969)]
2024-12-10T15:34:49,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:49,201 DEBUG [d6bcd7308055:34879 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T15:34:49,202 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6bcd7308055,44261,1733844888357
2024-12-10T15:34:49,204 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44261,1733844888357, state=OPENING
2024-12-10T15:34:49,257 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T15:34:49,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:49,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:49,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:49,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:49,269 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T15:34:49,269 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44261,1733844888357}]
2024-12-10T15:34:49,424 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T15:34:49,428 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59895, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T15:34:49,434 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T15:34:49,435 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:49,437 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6bcd7308055%2C44261%2C1733844888357.meta, suffix=.meta, logDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/d6bcd7308055,44261,1733844888357, archiveDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs, maxLogs=32
2024-12-10T15:34:49,438 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d6bcd7308055%2C44261%2C1733844888357.meta.1733844889437.meta
2024-12-10T15:34:49,444 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/d6bcd7308055,44261,1733844888357/d6bcd7308055%2C44261%2C1733844888357.meta.1733844889437.meta
2024-12-10T15:34:49,446 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45913:45913),(127.0.0.1/127.0.0.1:45969:45969)]
2024-12-10T15:34:49,447 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T15:34:49,447 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T15:34:49,448 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T15:34:49,448 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T15:34:49,448 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T15:34:49,448 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T15:34:49,448 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T15:34:49,448 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T15:34:49,450 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T15:34:49,451 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T15:34:49,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:49,451 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:49,452 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T15:34:49,452 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T15:34:49,453 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:49,453 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:49,453 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T15:34:49,454 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T15:34:49,454 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:49,455 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:49,455 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T15:34:49,456 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T15:34:49,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T15:34:49,457 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T15:34:49,457 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T15:34:49,458 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740
2024-12-10T15:34:49,460 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740
2024-12-10T15:34:49,461 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T15:34:49,461 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T15:34:49,462 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T15:34:49,463 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T15:34:49,463 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771001, jitterRate=-0.019622594118118286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T15:34:49,463 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T15:34:49,464 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733844889448Writing region info on filesystem at 1733844889448Initializing all the Stores at 1733844889449 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844889449Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844889450 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733844889450Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733844889450Cleaning up temporary data from old regions at 1733844889461 (+11 ms)Running coprocessor post-open hooks at 1733844889463 (+2 ms)Region opened successfully at 1733844889464 (+1 ms)
2024-12-10T15:34:49,464 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733844889424
2024-12-10T15:34:49,466 DEBUG [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T15:34:49,466 INFO  [RS_OPEN_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T15:34:49,466 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6bcd7308055,44261,1733844888357
2024-12-10T15:34:49,467 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6bcd7308055,44261,1733844888357, state=OPEN
2024-12-10T15:34:49,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:34:49,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T15:34:49,509 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6bcd7308055,44261,1733844888357
2024-12-10T15:34:49,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:49,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T15:34:49,514 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T15:34:49,514 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6bcd7308055,44261,1733844888357 in 240 msec
2024-12-10T15:34:49,519 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T15:34:49,519 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec
2024-12-10T15:34:49,520 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T15:34:49,520 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T15:34:49,521 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:34:49,521 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44261,1733844888357, seqNum=-1]
2024-12-10T15:34:49,521 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:34:49,522 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38501, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:34:49,527 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 767 msec
2024-12-10T15:34:49,527 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733844889527, completionTime=-1
2024-12-10T15:34:49,527 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T15:34:49,527 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T15:34:49,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T15:34:49,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733844949529
2024-12-10T15:34:49,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733845009529
2024-12-10T15:34:49,529 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6bcd7308055:34879, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,530 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,531 DEBUG [master/d6bcd7308055:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.118sec
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T15:34:49,533 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T15:34:49,534 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T15:34:49,534 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T15:34:49,536 DEBUG [master/d6bcd7308055:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T15:34:49,536 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T15:34:49,536 INFO  [master/d6bcd7308055:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6bcd7308055,34879,1733844888017-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T15:34:49,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.meta.1733844696659.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:49,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c7b74a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:49,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6bcd7308055,34879,-1 for getting cluster id
2024-12-10T15:34:49,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T15:34:49,583 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd7de8344-98fc-4665-8a72-ff1f87faa6cd'
2024-12-10T15:34:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T15:34:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d7de8344-98fc-4665-8a72-ff1f87faa6cd"

2024-12-10T15:34:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413b046d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6bcd7308055,34879,-1]
2024-12-10T15:34:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T15:34:49,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,587 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T15:34:49,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5f0305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T15:34:49,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T15:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6bcd7308055,44261,1733844888357, seqNum=-1]
2024-12-10T15:34:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T15:34:49,592 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T15:34:49,594 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6bcd7308055,34879,1733844888017
2024-12-10T15:34:49,594 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T15:34:49,596 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T15:34:49,597 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T15:34:49,599 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1, archiveDir=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs, maxLogs=32
2024-12-10T15:34:49,599 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733844889599
2024-12-10T15:34:49,604 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1/test.com%2C8080%2C1.1733844889599
2024-12-10T15:34:49,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45969:45969),(127.0.0.1/127.0.0.1:45913:45913)]
2024-12-10T15:34:49,605 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733844889605
2024-12-10T15:34:49,613 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,613 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,614 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,614 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,614 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,614 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1/test.com%2C8080%2C1.1733844889599 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1/test.com%2C8080%2C1.1733844889605
2024-12-10T15:34:49,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45969:45969),(127.0.0.1/127.0.0.1:45913:45913)]
2024-12-10T15:34:49,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1/test.com%2C8080%2C1.1733844889599 is not closed yet, will try archiving it next time
2024-12-10T15:34:49,616 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,616 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,617 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741835_1011 (size=93)
2024-12-10T15:34:49,617 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741835_1011 (size=93)
2024-12-10T15:34:49,617 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,618 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/WALs/test.com,8080,1/test.com%2C8080%2C1.1733844889599 to hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs/test.com%2C8080%2C1.1733844889599
2024-12-10T15:34:49,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741836_1012 (size=93)
2024-12-10T15:34:49,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741836_1012 (size=93)
2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs
2024-12-10T15:34:49,620 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733844889605)
2024-12-10T15:34:49,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T15:34:49,620 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,620 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T15:34:49,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=486117536, stopped=false
2024-12-10T15:34:49,620 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6bcd7308055,34879,1733844888017
2024-12-10T15:34:49,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:49,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T15:34:49,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:49,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:49,636 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:49,636 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T15:34:49,636 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:49,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6bcd7308055,44261,1733844888357' *****
2024-12-10T15:34:49,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T15:34:49,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T15:34:49,637 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(959): stopping server d6bcd7308055,44261,1733844888357
2024-12-10T15:34:49,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6bcd7308055:44261.
2024-12-10T15:34:49,637 DEBUG [RS:0;d6bcd7308055:44261 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T15:34:49,637 DEBUG [RS:0;d6bcd7308055:44261 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T15:34:49,637 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T15:34:49,638 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T15:34:49,638 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-10T15:34:49,638 DEBUG [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T15:34:49,638 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T15:34:49,638 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T15:34:49,638 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T15:34:49,638 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T15:34:49,638 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T15:34:49,638 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-10T15:34:49,655 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/.tmp/ns/6ea82e8d28ab44228653a1e1079388d7 is 43, key is default/ns:d/1733844889523/Put/seqid=0
2024-12-10T15:34:49,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741837_1013 (size=5153)
2024-12-10T15:34:49,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741837_1013 (size=5153)
2024-12-10T15:34:49,660 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/.tmp/ns/6ea82e8d28ab44228653a1e1079388d7
2024-12-10T15:34:49,665 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/.tmp/ns/6ea82e8d28ab44228653a1e1079388d7 as hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/ns/6ea82e8d28ab44228653a1e1079388d7
2024-12-10T15:34:49,669 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/ns/6ea82e8d28ab44228653a1e1079388d7, entries=2, sequenceid=6, filesize=5.0 K
2024-12-10T15:34:49,670 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=6, compaction requested=false
2024-12-10T15:34:49,673 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-10T15:34:49,674 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T15:34:49,674 INFO  [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:49,674 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733844889638Running coprocessor pre-close hooks at 1733844889638Disabling compacts and flushes for region at 1733844889638Disabling writes for close at 1733844889638Obtaining lock to block concurrent updates at 1733844889638Preparing flush snapshotting stores in 1588230740 at 1733844889638Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733844889638Flushing stores of hbase:meta,,1.1588230740 at 1733844889639 (+1 ms)Flushing 1588230740/ns: creating writer at 1733844889639Flushing 1588230740/ns: appending metadata at 1733844889655 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733844889655Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2127b2b5: reopening flushed file at 1733844889664 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=6, compaction requested=false at 1733844889670 (+6 ms)Writing region close event to WAL at 1733844889670Running coprocessor post-close hooks at 1733844889674 (+4 ms)Closed at 1733844889674
2024-12-10T15:34:49,674 DEBUG [RS_CLOSE_META-regionserver/d6bcd7308055:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T15:34:49,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/WALs/d6bcd7308055,42517,1733844695553/d6bcd7308055%2C42517%2C1733844695553.1733844696295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:49,838 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(976): stopping server d6bcd7308055,44261,1733844888357; all regions closed.
2024-12-10T15:34:49,839 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,840 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,840 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,840 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,841 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741834_1010 (size=1152)
2024-12-10T15:34:49,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741834_1010 (size=1152)
2024-12-10T15:34:49,850 DEBUG [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs
2024-12-10T15:34:49,850 INFO  [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44261%2C1733844888357.meta:.meta(num 1733844889437)
2024-12-10T15:34:49,851 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,851 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,851 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,851 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,851 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:49,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741833_1009 (size=93)
2024-12-10T15:34:49,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741833_1009 (size=93)
2024-12-10T15:34:49,857 DEBUG [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/oldWALs
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d6bcd7308055%2C44261%2C1733844888357:(num 1733844888954)
2024-12-10T15:34:49,857 DEBUG [RS:0;d6bcd7308055:44261 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.ChoreService(370): Chore service for: regionserver/d6bcd7308055:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:49,857 INFO  [regionserver/d6bcd7308055:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:49,857 INFO  [RS:0;d6bcd7308055:44261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44261
2024-12-10T15:34:49,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T15:34:49,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6bcd7308055,44261,1733844888357
2024-12-10T15:34:49,867 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:49,878 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6bcd7308055,44261,1733844888357]
2024-12-10T15:34:49,888 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6bcd7308055,44261,1733844888357 already deleted, retry=false
2024-12-10T15:34:49,888 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6bcd7308055,44261,1733844888357 expired; onlineServers=0
2024-12-10T15:34:49,888 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6bcd7308055,34879,1733844888017' *****
2024-12-10T15:34:49,888 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T15:34:49,888 INFO  [M:0;d6bcd7308055:34879 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T15:34:49,888 INFO  [M:0;d6bcd7308055:34879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T15:34:49,888 DEBUG [M:0;d6bcd7308055:34879 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T15:34:49,889 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T15:34:49,889 DEBUG [M:0;d6bcd7308055:34879 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T15:34:49,889 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844888769 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.large.0-1733844888769,5,FailOnTimeoutGroup]
2024-12-10T15:34:49,889 DEBUG [master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844888769 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6bcd7308055:0:becomeActiveMaster-HFileCleaner.small.0-1733844888769,5,FailOnTimeoutGroup]
2024-12-10T15:34:49,889 INFO  [M:0;d6bcd7308055:34879 {}] hbase.ChoreService(370): Chore service for: master/d6bcd7308055:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T15:34:49,889 INFO  [M:0;d6bcd7308055:34879 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T15:34:49,889 DEBUG [M:0;d6bcd7308055:34879 {}] master.HMaster(1795): Stopping service threads
2024-12-10T15:34:49,889 INFO  [M:0;d6bcd7308055:34879 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T15:34:49,889 INFO  [M:0;d6bcd7308055:34879 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T15:34:49,889 INFO  [M:0;d6bcd7308055:34879 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T15:34:49,889 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T15:34:49,899 DEBUG [M:0;d6bcd7308055:34879 {}] zookeeper.ZKUtil(347): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T15:34:49,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T15:34:49,899 WARN  [M:0;d6bcd7308055:34879 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T15:34:49,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T15:34:49,913 INFO  [M:0;d6bcd7308055:34879 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/.lastflushedseqids
2024-12-10T15:34:49,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741838_1014 (size=99)
2024-12-10T15:34:49,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741838_1014 (size=99)
2024-12-10T15:34:49,919 INFO  [M:0;d6bcd7308055:34879 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T15:34:49,919 INFO  [M:0;d6bcd7308055:34879 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T15:34:49,920 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T15:34:49,920 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:49,920 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:49,920 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T15:34:49,920 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:49,920 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-10T15:34:49,935 DEBUG [M:0;d6bcd7308055:34879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2f435d63910412db005e1aa6bd3b96a is 82, key is hbase:meta,,1/info:regioninfo/1733844889466/Put/seqid=0
2024-12-10T15:34:49,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741839_1015 (size=5672)
2024-12-10T15:34:49,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741839_1015 (size=5672)
2024-12-10T15:34:49,940 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2f435d63910412db005e1aa6bd3b96a
2024-12-10T15:34:49,956 DEBUG [M:0;d6bcd7308055:34879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fac3828401d421282e176dcdb0e436a is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733844889526/Put/seqid=0
2024-12-10T15:34:49,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741840_1016 (size=5275)
2024-12-10T15:34:49,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741840_1016 (size=5275)
2024-12-10T15:34:49,960 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fac3828401d421282e176dcdb0e436a
2024-12-10T15:34:49,977 DEBUG [M:0;d6bcd7308055:34879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2a556f3ccc24c56bdede1d59f130b73 is 69, key is d6bcd7308055,44261,1733844888357/rs:state/1733844888808/Put/seqid=0
2024-12-10T15:34:49,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:49,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44261-0x100105d708b0001, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:49,978 INFO  [RS:0;d6bcd7308055:44261 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:49,978 INFO  [RS:0;d6bcd7308055:44261 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6bcd7308055,44261,1733844888357; zookeeper connection closed.
2024-12-10T15:34:49,978 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47968aca {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47968aca
2024-12-10T15:34:49,978 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T15:34:49,981 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741841_1017 (size=5156)
2024-12-10T15:34:49,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741841_1017 (size=5156)
2024-12-10T15:34:49,982 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2a556f3ccc24c56bdede1d59f130b73
2024-12-10T15:34:49,997 DEBUG [M:0;d6bcd7308055:34879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fdeaf6a54c1349f7a157e61187a3c1c4 is 52, key is load_balancer_on/state:d/1733844889596/Put/seqid=0
2024-12-10T15:34:50,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741842_1018 (size=5056)
2024-12-10T15:34:50,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741842_1018 (size=5056)
2024-12-10T15:34:50,002 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fdeaf6a54c1349f7a157e61187a3c1c4
2024-12-10T15:34:50,006 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2f435d63910412db005e1aa6bd3b96a as hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2f435d63910412db005e1aa6bd3b96a
2024-12-10T15:34:50,010 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2f435d63910412db005e1aa6bd3b96a, entries=8, sequenceid=29, filesize=5.5 K
2024-12-10T15:34:50,010 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fac3828401d421282e176dcdb0e436a as hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fac3828401d421282e176dcdb0e436a
2024-12-10T15:34:50,014 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fac3828401d421282e176dcdb0e436a, entries=3, sequenceid=29, filesize=5.2 K
2024-12-10T15:34:50,014 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2a556f3ccc24c56bdede1d59f130b73 as hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a2a556f3ccc24c56bdede1d59f130b73
2024-12-10T15:34:50,017 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a2a556f3ccc24c56bdede1d59f130b73, entries=1, sequenceid=29, filesize=5.0 K
2024-12-10T15:34:50,018 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fdeaf6a54c1349f7a157e61187a3c1c4 as hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fdeaf6a54c1349f7a157e61187a3c1c4
2024-12-10T15:34:50,022 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46409/user/jenkins/test-data/904826eb-3c0d-c715-fd63-335e896c41ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fdeaf6a54c1349f7a157e61187a3c1c4, entries=1, sequenceid=29, filesize=4.9 K
2024-12-10T15:34:50,023 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=29, compaction requested=false
2024-12-10T15:34:50,024 INFO  [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T15:34:50,024 DEBUG [M:0;d6bcd7308055:34879 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733844889920Disabling compacts and flushes for region at 1733844889920Disabling writes for close at 1733844889920Obtaining lock to block concurrent updates at 1733844889920Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733844889920Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733844889920Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733844889921 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733844889921Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733844889935 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733844889935Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733844889944 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733844889956 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733844889956Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733844889964 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733844889977 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733844889977Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733844889985 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733844889997 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733844889997Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@203f7fcb: reopening flushed file at 1733844890006 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cae547f: reopening flushed file at 1733844890010 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6558eac8: reopening flushed file at 1733844890014 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b57a853: reopening flushed file at 1733844890018 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=29, compaction requested=false at 1733844890023 (+5 ms)Writing region close event to WAL at 1733844890024 (+1 ms)Closed at 1733844890024
2024-12-10T15:34:50,024 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:50,024 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:50,024 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:50,024 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:50,025 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T15:34:50,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42797 is added to blk_1073741830_1006 (size=10311)
2024-12-10T15:34:50,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44905 is added to blk_1073741830_1006 (size=10311)
2024-12-10T15:34:50,027 INFO  [M:0;d6bcd7308055:34879 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T15:34:50,027 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T15:34:50,027 INFO  [M:0;d6bcd7308055:34879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34879
2024-12-10T15:34:50,027 INFO  [M:0;d6bcd7308055:34879 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T15:34:50,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34973/user/jenkins/test-data/4ab8e9d2-d276-9004-8642-24fb747fd165/MasterData/WALs/d6bcd7308055,44557,1733844695385/d6bcd7308055%2C44557%2C1733844695385.1733844695698
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor107.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T15:34:50,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:50,136 INFO  [M:0;d6bcd7308055:34879 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T15:34:50,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34879-0x100105d708b0000, quorum=127.0.0.1:60588, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T15:34:50,140 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ed6212e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:50,141 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3593a6d0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:50,141 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:50,141 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cde2eb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:50,141 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f89e80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:50,143 WARN  [BP-2090031351-172.17.0.2-1733844885923 heartbeating to localhost/127.0.0.1:46409 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:50,143 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:50,143 WARN  [BP-2090031351-172.17.0.2-1733844885923 heartbeating to localhost/127.0.0.1:46409 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2090031351-172.17.0.2-1733844885923 (Datanode Uuid 32584107-aadf-4dde-a866-8777bc8e5832) service to localhost/127.0.0.1:46409
2024-12-10T15:34:50,143 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:50,144 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data3/current/BP-2090031351-172.17.0.2-1733844885923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:50,145 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data4/current/BP-2090031351-172.17.0.2-1733844885923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:50,145 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:50,147 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43d2c6e8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T15:34:50,148 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@55c02843{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:50,148 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:50,148 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2685eb00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:50,148 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1194c52d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:50,149 WARN  [BP-2090031351-172.17.0.2-1733844885923 heartbeating to localhost/127.0.0.1:46409 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T15:34:50,149 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T15:34:50,149 WARN  [BP-2090031351-172.17.0.2-1733844885923 heartbeating to localhost/127.0.0.1:46409 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2090031351-172.17.0.2-1733844885923 (Datanode Uuid 661dabbf-646d-4d4e-913d-150c73bc172e) service to localhost/127.0.0.1:46409
2024-12-10T15:34:50,149 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T15:34:50,150 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data1/current/BP-2090031351-172.17.0.2-1733844885923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:50,150 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/cluster_f00abfe0-d06f-ecd1-4da8-3805e3a0941c/data/data2/current/BP-2090031351-172.17.0.2-1733844885923 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T15:34:50,150 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T15:34:50,156 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@69ffafd5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T15:34:50,156 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29afca44{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T15:34:50,156 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T15:34:50,156 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@117d3f54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T15:34:50,157 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48174132{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de086c4-7120-511e-0ec0-fdd5976315df/hadoop.log.dir/,STOPPED}
2024-12-10T15:34:50,162 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T15:34:50,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T15:34:50,184 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=279 (was 240)
Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:46409 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46409
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46409
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:46409
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46409
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:46409 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46409
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (599498808) connection to localhost/127.0.0.1:46409 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 200), ProcessCount=11 (was 11), AvailableMemoryMB=7208 (was 7213)